From 42c47329fcc644093ef91686b9ce3000b232a127 Mon Sep 17 00:00:00 2001 From: Rawan Herzallah Date: Mon, 4 Apr 2016 14:24:01 +0000 Subject: [PATCH] Implement ConnectX-4 configurations Add changes to implement features for ConnectX-4 adapters Change-Id: Ie66162a2cb37ea3c1766f20bf218fb239eca8cf3 --- deployment_scripts/common | 4 +- deployment_scripts/configure_qos.sh | 7 +- deployment_scripts/install_ofed.sh | 41 ++----- deployment_scripts/mellanox_settings.py | 79 ++++++++++-- .../post_set_mellanox_interfaces.sh | 32 +++-- .../puppet/manifests/compute.pp | 5 +- .../puppet/manifests/controller.pp | 3 +- .../puppet/manifests/iser_rename.pp | 2 +- .../manifests/compute_sriov.pp | 3 +- .../manifests/controller_sriov.pp | 3 +- .../templates/iser_rename.erb | 11 +- deployment_scripts/sriov.sh | 116 +++++++++++++----- deployment_scripts/vxlan_offloading.sh | 14 ++- deployment_tasks.yaml | 34 ++--- environment_config.yaml | 2 +- metadata.yaml | 13 +- pre_build_hook | 2 +- 17 files changed, 242 insertions(+), 129 deletions(-) diff --git a/deployment_scripts/common b/deployment_scripts/common index 3c1f61a..a2bf050 100755 --- a/deployment_scripts/common +++ b/deployment_scripts/common @@ -86,7 +86,9 @@ readonly USER_NUM_OF_VFS=`get_mlnx_param num_of_vfs` readonly ISER=`get_mlnx_param iser` readonly MAX_VFS=62 readonly MIN_VFS=1 -readonly CX='ConnectX3' +readonly CX=`get_mlnx_param cx_card` +readonly NETWORK_TYPE=`get_mlnx_param network_type` +readonly PHYSICAL_PORT=`get_mlnx_param physical_port` readonly VXLAN_OFFLOADING=`get_mlnx_param vxlan_offloading` readonly ROLES=`get_param roles` readonly ROLE=`get_param role` diff --git a/deployment_scripts/configure_qos.sh b/deployment_scripts/configure_qos.sh index 2750726..8180c08 100755 --- a/deployment_scripts/configure_qos.sh +++ b/deployment_scripts/configure_qos.sh @@ -64,7 +64,12 @@ function configure_qos () { case $SCRIPT_MODE in 'configure') - configure_qos + if [ "$CX" == "ConnectX-3" ]; then + configure_qos + fi + if [ "$CX" == "ConnectX-4" ]; then + logger_print info "QoS is not implemented for ConnectX-4." + fi ;; *) logger_print error "Unsupported execution mode ${SCRIPT_MODE}." diff --git a/deployment_scripts/install_ofed.sh b/deployment_scripts/install_ofed.sh index 7b23082..df7da01 100755 --- a/deployment_scripts/install_ofed.sh +++ b/deployment_scripts/install_ofed.sh @@ -87,7 +87,8 @@ function install_ofed_without_fw_update () { logger_print info "Installing OFED drivers" OFED_INSTALL_SCRIPT_CMD="/usr/bin/perl ${OFED_INSTALL_SCRIPT}" - ${OFED_INSTALL_SCRIPT_CMD} --force --enable-sriov --without-fw-update + ${OFED_INSTALL_SCRIPT_CMD} --force --without-fw-update + rc=$? if [ $rc -ne 0 ]; then logger_print error "Failed execute ${OFED_INSTALL_SCRIPT_CMD} error code ${rc}" @@ -97,36 +98,8 @@ function install_ofed_without_fw_update () { fi } -function update_fw_if_not_oem () { - BUS_ID=`lspci | grep -m 1 Mellanox | cut -d' ' -f1` - if [ -z $BUS_ID ]; then - logger_print info "Didn't find bus, skipping firmware upgrade" - exit 0 - fi - - mstflint -d ${BUS_ID} q | grep -i PSID | grep MT_ - if [ $? -ne 0 ]; then - logger_print info "Not Mellanox Card, skipping firmware upgrade" - exit 0 - fi - - OFED_INSTALL_SCRIPT="${OFED_DIR}/mlnxofedinstall" - if [ ! -f $OFED_INSTALL_SCRIPT ]; then - logger_print error "Failed to find $OFED_INSTALL_SCRIPT" - exit 1 - fi - - logger_print info "Updating FW on Mellanox HCA with BUS ID = ${BUS_ID}" - - OFED_INSTALL_SCRIPT_CMD="/usr/bin/perl ${OFED_INSTALL_SCRIPT}" - ${OFED_INSTALL_SCRIPT_CMD} --force --enable-sriov --fw-update-only - if [ $? -ne 0 ]; then - logger_print error "Failed execute ${OFED_INSTALL_SCRIPT_CMD} error code $?" - fi -} - function enable_eipoib (){ - if [ $DRIVER == 'eth_ipoib' ]; then + if [ $NETWORK_TYPE == 'infiniband' ]; then sed -i s/^E_IPOIB_LOAD.*$/E_IPOIB_LOAD=yes/g /etc/infiniband/openib.conf # Set Buffers size @@ -161,9 +134,9 @@ if ! is_ofed_installed; then enable_eipoib fi -# OEM cards require a different dedicated OFED build, this build doesn't -# support them. -update_fw_if_not_oem - # Decrease loglevels for prevent flooding kernel messages to console sysctl_conf set 'kernel.printk' '4 4 1 7' +service openibd stop +service openibd start + +exit 0 diff --git a/deployment_scripts/mellanox_settings.py b/deployment_scripts/mellanox_settings.py index 6219bdd..9b4a034 100755 --- a/deployment_scripts/mellanox_settings.py +++ b/deployment_scripts/mellanox_settings.py @@ -16,6 +16,7 @@ import os import sys +import subprocess import yaml import glob import logging @@ -24,7 +25,16 @@ import traceback MLNX_SECTION = 'mellanox-plugin' SETTINGS_FILE = '/etc/astute.yaml' PLUGIN_OVERRIDE_FILE = '/etc/hiera/override/plugins.yaml' -MLNX_DRIVERS_LIST = ('mlx4_en', 'eth_ipoib') +MLNX_DRIVERS_LIST = { 'ConnectX-3': {'eth_driver' : 'mlx4_en', 'ib_driver' : 'eth_ipoib'}, + 'ConnectX-4': {'eth_driver' : 'mlx5_core', 'ib_driver' : 'eth_ipoib'}} +MLNX_DRIVERS = set([MLNX_DRIVERS_LIST[card][net] + for card in MLNX_DRIVERS_LIST + for net in MLNX_DRIVERS_LIST[card]]) +ETH_DRIVERS = set([MLNX_DRIVERS_LIST[card][net] + for card in MLNX_DRIVERS_LIST + for net in MLNX_DRIVERS_LIST[card] + if net == 'eth_driver']) +IB_DRIVERS = MLNX_DRIVERS - ETH_DRIVERS ISER_IFC_NAME = 'mlnx_iser0' LOG_FILE = '/var/log/mellanox-plugin.log' @@ -63,6 +73,56 @@ class MellanoxSettings(object): ifc = mlnx_interfaces_section[network]['interface'] return ifc + @classmethod + def get_card_type(cls, driver): + for card in MLNX_DRIVERS_LIST.keys(): + if driver in MLNX_DRIVERS_LIST[card].values(): + network_driver_type = MLNX_DRIVERS_LIST[card].keys()[MLNX_DRIVERS_LIST[card].values()\ + .index(driver)] + return card + + @classmethod + def add_cx_card(cls): + mlnx_interfaces = cls.mlnx_interfaces_section + drivers = list() + interfaces = list() + mlnx = cls.get_mlnx_section() + for network_type, ifc_dict in mlnx_interfaces.iteritems(): + if 'driver' in ifc_dict and network_type in ['private','management','storage']: + drivers.append(ifc_dict['driver']) + interfaces.append(ifc_dict['interface']) + + drivers_set = list(set(drivers)) + interfaces_set = list(set(interfaces)) + if (len(drivers_set) > 1): + logging.error("Multiple ConnectX adapters was found in this environment.") + raise MellanoxSettingsException( + "Multiple ConnectX adapters was found in this environment." + ) + else: + current_driver = drivers_set[0] + mellanox_interface = interfaces_set[0] + if current_driver in ETH_DRIVERS: + mlnx['network_type'] = 'ethernet' + mlnx['cx_card'] = cls.get_card_type(current_driver) + elif current_driver in IB_DRIVERS: + mlnx['network_type'] = 'infiniband' + ibdev = os.popen('ibdev2netdev').readlines() + if not ibdev: + mlnx['cx_card'] = 'none' + logging.error('Failed executing ibdev2netdev') + return 0 + interface_line = [l for l in ibdev if mellanox_interface in l] + if interface_line and 'mlx5' in interface_line.pop(): + mlnx['cx_card'] = 'ConnectX-4' + else: + mlnx['cx_card'] = 'ConnectX-3' + + network_info_msg = 'Detected Network Type is: %s ', mlnx['network_type'] + card_info_msg = 'Detected Card Type is: %s ', mlnx['cx_card'] + logging.info(network_info_msg) + logging.info(card_info_msg) + @classmethod def add_driver(cls): interfaces = cls.get_interfaces_section() @@ -87,14 +147,14 @@ class MellanoxSettings(object): mlnx = cls.get_mlnx_section() private_ifc = cls.get_interface_by_network('private') - if mlnx['driver'] == 'eth_ipoib': + if mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['ib_driver']: if 'bus_info' not in interfaces[private_ifc]['vendor_specific']: raise MellanoxSettingsException( "Couldn't find 'bus_info' for interface " "{0}".format(private_ifc) ) mlnx['physical_port'] = interfaces[private_ifc]['vendor_specific']['bus_info'] - elif mlnx['driver'] == 'mlx4_en': + elif mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['eth_driver']: mlnx['physical_port'] = private_ifc @classmethod @@ -110,7 +170,7 @@ class MellanoxSettings(object): raise MellanoxSettingsException( "Failed reading vlan for br-storage" ) - if mlnx['driver'] == 'eth_ipoib': + if mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['ib_driver']: pkey = format((int(vlan) ^ 0x8000),'04x') mlnx['storage_pkey'] = pkey @@ -124,9 +184,9 @@ class MellanoxSettings(object): def add_iser_interface_name(cls): mlnx = cls.get_mlnx_section() storage_ifc = cls.get_interface_by_network('storage') - if mlnx['driver'] == 'mlx4_en': + if mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['eth_driver']: mlnx['iser_ifc_name'] = ISER_IFC_NAME - elif mlnx['driver'] == 'eth_ipoib': + elif mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['ib_driver']: interfaces = cls.get_interfaces_section() mlnx['iser_ifc_name'] = interfaces[storage_ifc]['vendor_specific']['bus_info'] else: @@ -150,7 +210,7 @@ class MellanoxSettings(object): # Handle iSER interface with and w/o vlan tagging storage_vlan = mlnx.get('storage_vlan') storage_parent = cls.get_interface_by_network('storage') - if storage_vlan and mlnx['driver'] == 'mlx4_en': # Use VLAN dev + if storage_vlan and mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['eth_driver']: # Use VLAN dev vlan_name = "{0}.{1}".format(ISER_IFC_NAME, storage_vlan) # Set storage rule to iSER interface vlan interface cls.data['network_scheme']['roles']['storage'] = vlan_name @@ -212,9 +272,10 @@ class MellanoxSettings(object): # the main change will be here because it reads phy_interfaces mlnx_interfaces = cls.mlnx_interfaces_section drivers = list() + mlnx = cls.get_mlnx_section() for network_type, ifc_dict in mlnx_interfaces.iteritems(): if 'driver' in ifc_dict and \ - ifc_dict['driver'] in MLNX_DRIVERS_LIST: + ifc_dict['driver'] in MLNX_DRIVERS_LIST[mlnx['cx_card']].values(): drivers.append(ifc_dict['driver']) return list(set(drivers)) @@ -232,6 +293,8 @@ class MellanoxSettings(object): @classmethod def update_role_settings(cls): + # detect ConnectX card + cls.add_cx_card() # realize the driver in use (eth/ib) cls.add_driver() # decide the physical function for SR-IOV diff --git a/deployment_scripts/post_set_mellanox_interfaces.sh b/deployment_scripts/post_set_mellanox_interfaces.sh index b208922..88e72e8 100755 --- a/deployment_scripts/post_set_mellanox_interfaces.sh +++ b/deployment_scripts/post_set_mellanox_interfaces.sh @@ -21,22 +21,28 @@ source ./common if ([[ $ROLES == *compute* ]] && [[ ! $ROLES == "compute" ]]) \ && [ $SRIOV == true ] ; then - # Update VFs - ./sriov.sh configure + if [ $CX == 'ConnectX-3' ]; then - # Kill tgt daemons if exists - tgt_locks=`find /var/run/ -name tgtd* | wc -l` - if [ $tgt_locks -ne 0 ];then - \rm -f /var/run/tgtd* && killall -9 tgtd - service tgt stop + # Update VFs + ./sriov.sh configure + + # Kill tgt daemons if exists + tgt_locks=`find /var/run/ -name tgtd* | wc -l` + if [ $tgt_locks -ne 0 ];then + \rm -f /var/run/tgtd* && killall -9 tgtd + service tgt stop + fi + + service openibd restart && service openvswitch-switch restart + + if [ $tgt_locks -ne 0 ];then + service tgt start + fi fi - - service openibd restart && service openvswitch-switch restart - - if [ $tgt_locks -ne 0 ];then - service tgt start + if [ $CX == 'ConnectX-4' ]; then + service openibd restart && service openvswitch-switch restart + #sleep 10 fi - # Verify VFs ./sriov.sh validate diff --git a/deployment_scripts/puppet/manifests/compute.pp b/deployment_scripts/puppet/manifests/compute.pp index 3c2e229..663e097 100644 --- a/deployment_scripts/puppet/manifests/compute.pp +++ b/deployment_scripts/puppet/manifests/compute.pp @@ -5,7 +5,7 @@ $firewall_driver = 'neutron.agent.firewall.NoopFirewallDriver' $private_net = $quantum_settings['default_private_net'] $roles = hiera('roles') -if ( $mlnx_driver == 'mlx4_en' ){ +if ( $mlnx['network_type'] == 'ethernet' and $mlnx['iser'] ){ $exclude_vf = '0' } else { $exclude_vf = '' @@ -16,12 +16,13 @@ if ($mlnx['sriov']) { physnet => $quantum_settings['predefined_networks'][$private_net]['L2']['physnet'], physifc => $mlnx['physical_port'], mlnx_driver => $mlnx['driver'], + network_type => $mlnx['network_type'], firewall_driver => $firewall_driver, exclude_vf => $exclude_vf, } } -# Configure QoS for ETH +# Configure QoS for ConnectX3 ETH if ( $mlnx['driver'] == 'mlx4_en' and $mlnx['mlnx_qos'] ) { class { 'mellanox_openstack::configure_qos' : mlnx_sriov => $mlnx['sriov'], diff --git a/deployment_scripts/puppet/manifests/controller.pp b/deployment_scripts/puppet/manifests/controller.pp index 08fbdd2..a019107 100644 --- a/deployment_scripts/puppet/manifests/controller.pp +++ b/deployment_scripts/puppet/manifests/controller.pp @@ -4,7 +4,7 @@ $eswitch_apply_profile_patch = 'True' $mechanism_drivers = 'openvswitch' $roles = hiera('roles') -# Configure QoS for ETH +# Configure QoS for connectX3 ETH if ( $mlnx['driver'] == 'mlx4_en' and $mlnx['mlnx_qos'] ) { class { 'mellanox_openstack::configure_qos' : mlnx_sriov => $mlnx['sriov'], @@ -20,6 +20,7 @@ if ($mlnx['sriov']) { eswitch_apply_profile_patch => $eswitch_apply_profile_patch, mechanism_drivers => $mechanism_drivers, mlnx_driver => $mlnx['driver'], + network_type => $mlnx['network_type'], mlnx_sriov => $mlnx['sriov'], pci_vendor_devices => $pci_vendor_devices, agent_required => $agent_required, diff --git a/deployment_scripts/puppet/manifests/iser_rename.pp b/deployment_scripts/puppet/manifests/iser_rename.pp index ddd9149..3e5b0a9 100644 --- a/deployment_scripts/puppet/manifests/iser_rename.pp +++ b/deployment_scripts/puppet/manifests/iser_rename.pp @@ -1,6 +1,6 @@ $mlnx = hiera('mellanox-plugin') -if ($mlnx['iser'] and $mlnx['driver'] == 'mlx4_en') { +if ($mlnx['iser'] and $mlnx['network_type'] == 'ethernet') { class { 'mellanox_openstack::iser_rename' : storage_parent => $mlnx['storage_parent'], iser_interface_name => $mlnx['iser_ifc_name'], diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/manifests/compute_sriov.pp b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/compute_sriov.pp index 9e6dfd2..0c49145 100644 --- a/deployment_scripts/puppet/modules/mellanox_openstack/manifests/compute_sriov.pp +++ b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/compute_sriov.pp @@ -2,6 +2,7 @@ class mellanox_openstack::compute_sriov ( $physnet, $physifc, $mlnx_driver, + $network_type, $firewall_driver, $exclude_vf, ) { @@ -35,7 +36,7 @@ class mellanox_openstack::compute_sriov ( value => $firewall_driver, } - if ( $mlnx_driver == 'mlx4_en' ){ + if ( $network_type == 'ethernet' ){ package { $sriov_agent_package: ensure => installed, } diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/manifests/controller_sriov.pp b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/controller_sriov.pp index 2d6afa0..fceb7bc 100644 --- a/deployment_scripts/puppet/modules/mellanox_openstack/manifests/controller_sriov.pp +++ b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/controller_sriov.pp @@ -3,6 +3,7 @@ class mellanox_openstack::controller_sriov ( $eswitch_apply_profile_patch, $mechanism_drivers, $mlnx_driver, + $network_type, $mlnx_sriov, $pci_vendor_devices, $agent_required, @@ -30,7 +31,7 @@ class mellanox_openstack::controller_sriov ( $sdn_extra_mechanism_driver='' } - if ( $mlnx_driver == 'mlx4_en' ){ + if ( $network_type == 'ethernet' ){ $ml2_extra_mechanism_driver = "${sdn_extra_mechanism_driver}sriovnicswitch" neutron_plugin_ml2 { 'ml2/mechanism_drivers': value => "${ml2_extra_mechanism_driver},${mechanism_drivers}"; diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb b/deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb index 9bc9ac8..c957ea2 100644 --- a/deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb +++ b/deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb @@ -93,7 +93,16 @@ if [ $PROBED_PORT_NAME != $ISER_NAME ]; then fi # restart OFED modules for udev changes to take effect - modprobe -r mlx4_en && modprobe mlx4_en + modprobe -r mlx4_en + modprobe -r mlx5_ib && modprobe -r mlx5_core + udevadm control --reload-rules + udevadm trigger + modprobe mlx4_en + modprobe mlx5_ib && modprobe mlx5_core + echo "#!/bin/bash" > /etc/network/if-up.d/iser_ifc + echo "ifconfig $ISER_NAME up " >> /etc/network/if-up.d/iser_ifc + ifup --all + if [ $? -ne 0 ]; then logger_print error "Mellanox drivers restart failed." exit 1 diff --git a/deployment_scripts/sriov.sh b/deployment_scripts/sriov.sh index 347ca2f..d5787f5 100755 --- a/deployment_scripts/sriov.sh +++ b/deployment_scripts/sriov.sh @@ -26,16 +26,16 @@ readonly GRUB_FILE_CENTOS="/boot/grub/grub.conf" readonly GRUB_FILE_UBUNTU="/boot/grub/grub.cfg" function get_port_type() { - if [ $DRIVER == 'mlx4_en' ]; then + if [ $NETWORK_TYPE == 'ethernet' ]; then port_type=2 - elif [ $DRIVER == 'eth_ipoib' ]; then + else port_type=1 fi echo $port_type } function get_num_probe_vfs () { - if [ $DRIVER == 'mlx4_en' ]; then + if [ `get_port_type` -eq "2" ]; then probe_vfs=`calculate_total_vfs` else probe_vfs=0 @@ -65,7 +65,7 @@ function calculate_total_vfs () { fi # Set Ethernet RDMA storage network - if [ $ISER == true ] && [ $DRIVER == 'mlx4_en' ] \ + if [ $ISER == true ] && [ `get_port_type` -eq "2" ] \ && [ $num_of_vfs -eq 0 ]; then num_of_vfs=1 fi @@ -149,37 +149,42 @@ function set_kernel_params () { function burn_vfs_in_fw () { total_vfs=$1 - # required for mlxconfig to discover mlnx devices - service openibd start &>/dev/null - service mst start &>/dev/null - devices=$(mst status -v | grep $CX| grep pciconf | awk '{print $2}') - for dev in $devices; do - logger_print debug "device=$dev" - mlxconfig -d $dev q | grep SRIOV | awk '{print $2}' | grep $SRIOV_ENABLED_FLAG &>/dev/null - sriov_enabled=$? - current_num_of_vfs=`mlxconfig -d $dev q | grep NUM_OF_VFS | awk '{print $2}'` - if [ $sriov_enabled -eq 0 ] 2>/dev/null; then - logger_print debug "Detected SR-IOV is already enabled" - else - logger_print debug "Detected SR-IOV is disabled" - fi - if [[ ! "$total_vfs" == "$current_num_of_vfs" ]] 2>/dev/null; then - logger_print debug "Current allowed number of VFs is ${current_num_of_vfs}, required number is ${total_vfs}" - logger_print debug "Trying mlxconfig -y -d ${dev} s SRIOV_EN=1 NUM_OF_VFS=${total_vfs}" - mlxconfig -y -d $dev s SRIOV_EN=1 NUM_OF_VFS=$total_vfs 2>&1 >/dev/null - if [ $? -ne 0 ]; then - logger_print error "Failed changing number of VFs in FW for HCA ${dev}" + if [ $CX == 'ConnectX-3' ]; then + # required for mlxconfig to discover mlnx devices + service openibd start &>/dev/null + service mst start &>/dev/null + devices=$(mst status -v | grep $CX| grep pciconf | awk '{print $2}') + for dev in $devices; do + logger_print debug "device=$dev" + mlxconfig -d $dev q | grep SRIOV | awk '{print $2}' | grep $SRIOV_ENABLED_FLAG &>/dev/null + sriov_enabled=$? + current_num_of_vfs=`mlxconfig -d $dev q | grep NUM_OF_VFS | awk '{print $2}'` + if [ $sriov_enabled -eq 0 ] 2>/dev/null; then + logger_print debug "Detected SR-IOV is already enabled" + else + logger_print debug "Detected SR-IOV is disabled" fi - else - logger_print debug "Current number of VFs is correctly set to ${current_num_of_vfs} in FW." - fi - done - service mst stop &>/dev/null + if [[ ! "$total_vfs" == "$current_num_of_vfs" ]] 2>/dev/null; then + logger_print debug "Current allowed number of VFs is ${current_num_of_vfs}, required number is ${total_vfs}" + logger_print debug "Trying mlxconfig -y -d ${dev} s SRIOV_EN=1 NUM_OF_VFS=${total_vfs}" + mlxconfig -y -d $dev s SRIOV_EN=1 NUM_OF_VFS=$total_vfs 2>&1 >/dev/null + if [ $? -ne 0 ]; then + logger_print error "Failed changing number of VFs in FW for HCA ${dev}" + fi + else + logger_print debug "Current number of VFs is correctly set to ${current_num_of_vfs} in FW." + fi + done + service mst stop &>/dev/null + fi + if [ $CX == 'ConnectX-4' ]; then + logger_print debug "Skipping burning ConnectX-4 as it is burnt in bootstrap stage." + fi } function is_sriov_required () { [ $SRIOV == true ] || - ( [ $ISER == true ] && [ $DRIVER == 'mlx4_en' ] ) + ( [ $ISER == true ] && [ `get_port_type` -eq "2" ] ) return $? } @@ -195,9 +200,18 @@ function configure_sriov () { probe_vfs=`get_num_probe_vfs` port_type=`get_port_type` - set_modprobe_file $total_vfs && set_kernel_params && burn_vfs_in_fw $total_vfs + if [ $CX == 'ConnectX-3' ]; then + set_modprobe_file $total_vfs && + logger_print info "Detected: ConnectX-3 card" + fi + + if [ $CX == 'ConnectX-4' ]; then + set_sriov $total_vfs && + logger_print info "Detected: ConnectX-4 card" + fi + return $? else logger_print info "Skipping SR-IOV configuration" @@ -236,8 +250,15 @@ function validate_sriov () { logger_print error "Failed , trying to fallback to ${FALLBACK_NUM_VFS}" probe_vfs=`get_num_probe_vfs` port_type=`get_port_type` - set_modprobe_file $FALLBACK_NUM_VFS - service openibd restart &> /dev/null + + if [ $CX == 'ConnectX-3' ]; then + set_modprobe_file $FALLBACK_NUM_VFS + service openibd restart &> /dev/null + fi + if [ $CX == 'ConnectX-4' ]; then + set_sriov $FALLBACK_NUM_VFS + fi + current_num_vfs=`lspci | grep -i mellanox | grep -i virtual | wc -l` if [ $current_num_vfs -eq $FALLBACK_NUM_VFS ]; then logger_print info "Fallback to ${FALLBACK_NUM_VFS} succeeded" @@ -248,11 +269,38 @@ function validate_sriov () { fi } +function set_sriov () { + PORT_TYPE=`get_port_type` + TOTAL_VFS=$1 + device_up=$PHYSICAL_PORT + + if [ ${#device_up} -eq 0 ]; then + logger_print error "Failed to find mlx5 up ports in ibdev2netdev." + exit 1 + else + res=`echo 0 > /sys/class/net/${device_up}/device/mlx5_num_vfs` + res=`echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs` + if [ ! $? -eq 0 ]; then + logger_print error "Failed to write $TOTAL_VFS > /sys/class/net/${device_up}/device/mlx5_num_vfs" + exit 1 + fi + echo "#!/bin/bash" > /etc/network/if-up.d/sriov_vfs + echo "echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs" >> /etc/network/if-up.d/sriov_vfs + chmod +x /etc/network/if-up.d/sriov_vfs + ifup --all + if [ ! $? -eq 0 ]; then + logger_print error "Failed to write $TOTAL_VFS > /sys/class/net/${device_up}/device/mlx5_num_vfs" + exit 1 + else + logger_print debug "Configured total vfs ${TOTAL_VFS} on ${device_up}" + fi + fi +} ################# case $SCRIPT_MODE in 'configure') - configure_sriov + configure_sriov ;; 'validate') validate_sriov diff --git a/deployment_scripts/vxlan_offloading.sh b/deployment_scripts/vxlan_offloading.sh index 5b5b546..525e651 100755 --- a/deployment_scripts/vxlan_offloading.sh +++ b/deployment_scripts/vxlan_offloading.sh @@ -20,16 +20,16 @@ source $SCRIPT_DIR/common readonly SCRIPT_MODE=$1 function get_port_type() { - if [ $DRIVER == 'mlx4_en' ]; then + if [ $NETWORK_TYPE == 'ethernet' ]; then port_type=2 - elif [ $DRIVER == 'eth_ipoib' ]; then + else port_type=1 fi echo $port_type } function is_vxlan_offloading_required () { - [ $VXLAN_OFFLOADING == true ] && [ $DRIVER == 'mlx4_en' ] + [ $VXLAN_OFFLOADING == true ] && [ $NETWORK_TYPE == 'ethernet' ] return $? } @@ -69,7 +69,13 @@ function set_modprobe_file () { configure_vxlan_offloading case $SCRIPT_MODE in 'configure') - configure_vxlan_offloading + if [ "$CX" == "ConnectX3" ]; then + configure_vxlan_offloading + fi + if [ "$CX" == "ConnectX4" ]; then + logger_print info "Skipping VXLAN configuration because VXLAN is configured with ConnectX4." + fi + ;; 'validate') # to be added later. diff --git a/deployment_tasks.yaml b/deployment_tasks.yaml index 1f2b0ca..dd322c0 100644 --- a/deployment_tasks.yaml +++ b/deployment_tasks.yaml @@ -63,17 +63,27 @@ # Install OFED + FW upgrade - id: install_mlnx_ofed role: '*' - required_for: [configure_vxlan_offload] + required_for: [customize_mellanox_hiera_post] requires: [validate_kernel_devel] type: shell parameters: cmd: ./install_ofed.sh timeout: 5000 +# Add relevant settings for Mellanox manifests to mellanox plugin section in +# Hiera, to make the data easily accessible and independent of astute.yaml +- id: customize_mellanox_hiera_post + role: '*' + required_for: [configure_vxlan_offload] + requires: [install_mlnx_ofed] + type: shell + parameters: + cmd: ./mellanox_settings.py + timeout: 100 # add VXLAN offloading parameters - id: configure_vxlan_offload role: '*' required_for: [configure_sriov] - requires: [install_mlnx_ofed] + requires: [customize_mellanox_hiera_post] type: shell parameters: cmd: ./vxlan_offloading.sh configure @@ -87,34 +97,26 @@ type: shell parameters: cmd: ./sriov.sh configure - timeout: 200 + timeout: 300 # if VXLAN is not required. Rename iSER interface for Eth mode - id: rename_iser_probe_vf role: '*' - required_for: [reboot_after_ofed] + required_for: [validate_sriov] requires: [configure_sriov] type: puppet parameters: puppet_manifest: puppet/manifests/iser_rename.pp puppet_modules: puppet/modules:/etc/puppet/modules - timeout: 200 -# Reboot due to OFED installation / IOMMU configuration -- id: reboot_after_ofed - role: '*' - required_for: [validate_sriov] - requires: [rename_iser_probe_vf] - type: reboot - parameters: - timeout: 5000 + timeout: 300 # Check number of VFs - id: validate_sriov role: '*' required_for: [configure_qos] - requires: [reboot_after_ofed] + requires: [rename_iser_probe_vf] type: shell parameters: cmd: ./sriov.sh validate - timeout: 200 + timeout: 300 # Enable QoS if required - id: configure_qos role: '*' @@ -162,7 +164,7 @@ type: shell parameters: cmd: ./post_set_mellanox_interfaces.sh - timeout: 200 + timeout: 500 # Execute post_deployment manifest for each role - id: configure_mellanox_controller role: ['controller', 'primary-controller'] diff --git a/environment_config.yaml b/environment_config.yaml index 218a2f3..e9646d4 100644 --- a/environment_config.yaml +++ b/environment_config.yaml @@ -32,7 +32,7 @@ attributes: If selected, Neutron "Quality of Service" (QoS) will be enabled for SR-IOV ports over Mellanox HCAs. This feature is supported in Ethernet mode over Neutron with VLAN after SR-IOV direct port creation. weight: 30 - type: "checkbox" + type: hidden restrictions: - condition: "settings:common.libvirt_type.value != 'kvm' or not (settings:mellanox-plugin.sriov.value == true) or not (cluster:net_provider == 'neutron' and networking_parameters:segmentation_type == 'vlan')" message: "For Mellanox QoS support in Neutron, hypervisor type should be KVM and the environment networking should be based on Neutron with VLAN segmentation with Mellanox SR-IOV support." diff --git a/metadata.yaml b/metadata.yaml index 915750b..ee3534a 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -2,16 +2,16 @@ name: mellanox-plugin # Human-readable name for your plugin -title: Mellanox Openstack Features +title: Mellanox ConnectX-4 Openstack Features # Plugin version -version: 3.0.0 +version: 3.1.0 # Description -description: Enable features over Mellanox hardware +description: Enable features over Mellanox ConnectX-4 Adapters # Required fuel version -fuel_version: ['7.0', '8.0'] +fuel_version: ['8.0'] # Specify license of your plugin licenses: ['Apache License Version 2.0'] @@ -27,11 +27,6 @@ groups: ['storage::cinder', 'network', 'hypervisor'] # The plugin is compatible with releases in the list releases: - - os: ubuntu - version: 2015.1.0-7.0 - mode: ['ha'] - deployment_scripts_path: deployment_scripts/ - repository_path: repositories/ubuntu - os: ubuntu version: liberty-8.0 mode: ['ha'] diff --git a/pre_build_hook b/pre_build_hook index e7b851a..d22f900 100755 --- a/pre_build_hook +++ b/pre_build_hook @@ -72,7 +72,7 @@ old_debs="${PLUGIN_DIR}/repositories/ubuntu/*.deb" deb_files="cirros-testvm-mellanox_0.3.2-ubuntu3_amd64.deb cirros-testvm-mellanox-ib_0.3.2-9_amd64.deb eswitchd_1.0.0-18_amd64.deb - mlnx-ofed-fuel_3.1-1.5.5_amd64.deb + mlnx-ofed-fuel_3.2-2.0.0.0_amd64.deb mlnx-ofed-kernel-dkms_3.1-OFED.3.1.1.0.3.1.g9032737_all.deb mlnx-ofed-kernel-utils_3.1-OFED.3.1.1.0.3.1.g9032737_amd64.deb lldpd_0.9.1-0_amd64.deb