diff --git a/bootstrap/sync/opt/mellanox_init.sh b/bootstrap/sync/opt/mellanox_init.sh index 995520f..597923b 100755 --- a/bootstrap/sync/opt/mellanox_init.sh +++ b/bootstrap/sync/opt/mellanox_init.sh @@ -23,6 +23,9 @@ if [ ! -z $1 ]; then sed -i "1a\export LINK_TYPE=$1" /etc/rc.local fi +# Set MAX_NUM_VFS to run on boot +sed -i "1a\export MAX_NUM_VFS=$MAX_NUM_VFS" /etc/rc.local + # Install required packages dpkg -i ${OFED_DEBS_DIR}/mlnx-ofed-kernel-utils*.deb dpkg -i ${OFED_DEBS_DIR}/mlnx-ofed-kernel-dkms*.deb diff --git a/bootstrap/sync/usr/bin/mlnx_fw_upgrade b/bootstrap/sync/usr/bin/mlnx_fw_upgrade index 8b73a0a..c436db5 100755 --- a/bootstrap/sync/usr/bin/mlnx_fw_upgrade +++ b/bootstrap/sync/usr/bin/mlnx_fw_upgrade @@ -15,7 +15,7 @@ # limitations under the License. # Set defaults -MAX_VFS=16 +MAX_VFS=$MAX_NUM_VFS LOG_FILE=/var/log/mlnx_fw_update.log OFED_DEBS_DIR=/opt/ofed/MLNX_OFED/DEBS/ diff --git a/deployment_scripts/mellanox_settings.py b/deployment_scripts/mellanox_settings.py index 658198b..0822c59 100755 --- a/deployment_scripts/mellanox_settings.py +++ b/deployment_scripts/mellanox_settings.py @@ -22,6 +22,7 @@ import glob import logging import traceback +MAX_NUM_VFS = 16 MLNX_SECTION = 'mellanox-plugin' SETTINGS_FILE = '/etc/astute.yaml' PLUGIN_OVERRIDE_FILE = '/etc/hiera/override/plugins.yaml' @@ -291,6 +292,22 @@ class MellanoxSettings(object): def is_vxlan_offloading_enabled(cls): return cls.get_mlnx_section()['vxlan_offloading'] + @classmethod + def add_reboot_condition(cls): + # if MAX_NUM_VF > default which is 16, reboot + mlnx = cls.get_mlnx_section() + mst_start = os.popen('mst start;').readlines() + burned_num_vfs_list = list() + devices = os.popen('mst status -v| grep pciconf | grep {0} | awk \'{{print $2}}\' '.format( + mlnx['cx_card'].replace("-",""))).readlines() + for dev in devices: + num = os.popen('mlxconfig -d {0} q | grep NUM_OF_VFS | awk \'{{print $2}}\' \ + '.format(dev.rsplit()[0])).readlines() + burned_num_vfs_list.append(num[0].rsplit()[0]) + burned_num_vfs = list(set(burned_num_vfs_list))[0] + if burned_num_vfs > MAX_NUM_VFS or mlnx['num_of_vfs'] > MAX_NUM_VFS : + mlnx['reboot_required'] = True + @classmethod def update_role_settings(cls): # detect ConnectX card @@ -305,6 +322,8 @@ class MellanoxSettings(object): cls.add_storage_vlan() cls.add_iser_interface_name() cls.set_storage_networking_scheme() + # fill reboot condition + cls.add_reboot_condition() @classmethod def read_from_yaml(cls, settings_file): diff --git a/deployment_scripts/sriov.sh b/deployment_scripts/sriov.sh index a79c64b..22048ef 100755 --- a/deployment_scripts/sriov.sh +++ b/deployment_scripts/sriov.sh @@ -178,7 +178,18 @@ function burn_vfs_in_fw () { service mst stop &>/dev/null fi if [ $CX == 'ConnectX-4' ]; then - logger_print debug "Skipping burning ConnectX-4 as it is burnt in bootstrap stage." + # required for mlxconfig to discover mlnx devices + service openibd start &>/dev/null + service mst start &>/dev/null + devices=$(mst status -v | grep $CX| grep pciconf | awk '{print $2}') + for dev in $devices; do + current_fw_vfs=`mlxconfig -d $dev q | grep NUM_OF_VFS | awk '{print $2}'` + if [ "$total_vfs" -gt "$current_fw_vfs" ]; then + logger_print debug "device=$dev" + logger_print debug "Trying mlxconfig -d ${dev} -y set NUM_OF_VFS=${total_vfs}" + mlxconfig -d $dev -y set NUM_OF_VFS=$total_vfs + fi + done fi } diff --git a/deployment_tasks.yaml b/deployment_tasks.yaml index 86e5fa4..eabb268 100644 --- a/deployment_tasks.yaml +++ b/deployment_tasks.yaml @@ -34,7 +34,7 @@ # this script links astute.yaml to any .yaml on each node. - id: link_astute_file role: '*' - required_for: [customize_mellanox_hiera] + required_for: [validate_kernel_devel] requires: [init_mlnx_logs] type: shell parameters: @@ -44,7 +44,7 @@ - id: validate_kernel_devel role: '*' required_for: [install_mlnx_ofed] - requires: [customize_mellanox_hiera] + requires: [link_astute_file] type: puppet parameters: puppet_manifest: puppet/manifests/verify_kernel_devel.pp @@ -53,7 +53,7 @@ # Install OFED + FW upgrade - id: install_mlnx_ofed role: '*' - required_for: [customize_mellanox_hiera_post] + required_for: [customize_mellanox_hiera] requires: [validate_kernel_devel] type: shell parameters: @@ -61,7 +61,7 @@ timeout: 5000 # Add relevant settings for Mellanox manifests to mellanox plugin section in # Hiera, to make the data easily accessible and independent of astute.yaml -- id: customize_mellanox_hiera_post +- id: customize_mellanox_hiera role: '*' required_for: [configure_vxlan_offload] requires: [install_mlnx_ofed] @@ -73,7 +73,7 @@ - id: configure_vxlan_offload role: '*' required_for: [configure_sriov] - requires: [customize_mellanox_hiera_post] + requires: [customize_mellanox_hiera] type: shell parameters: cmd: ./vxlan_offloading.sh configure @@ -91,13 +91,22 @@ # if VXLAN is not required. Rename iSER interface for Eth mode - id: rename_iser_probe_vf role: '*' - required_for: [validate_sriov] + required_for: [reboot_after_max_vfs_num_change] requires: [configure_sriov] type: puppet parameters: puppet_manifest: puppet/manifests/iser_rename.pp puppet_modules: puppet/modules:/etc/puppet/modules timeout: 300 +# Reboot due to changing MAX number of VFs +- id: reboot_after_max_vfs_num_change + role: '*' + required_for: [validate_sriov] + requires: [rename_iser_probe_vf] + type: reboot + condition: "settings:mellanox-plugin.reboot_required.value == true" + parameters: + timeout: 6000 # Check number of VFs - id: validate_sriov role: '*' diff --git a/environment_config.yaml b/environment_config.yaml index af90a3f..b70a9a0 100644 --- a/environment_config.yaml +++ b/environment_config.yaml @@ -26,6 +26,14 @@ attributes: - condition: "settings:mellanox-plugin.sriov.value == false" message: "In order to change the number of virtual NICs, Neutron SR-IOV plugin should be checked." + reboot_required: + value: false + label: "Reboot when maximum number of vfs configured by the user is greater than the one burned in FW." + description: | + Reboots Slave nodes if the maximum number of vfs configured by the user is greator than the number burned in FW. + weight: 30 + type: hidden + mlnx_qos: value: false label: "Quality of Service over Mellanox SR-IOV ports (Neutron)" diff --git a/metadata.yaml b/metadata.yaml index b8ba0ca..df05e57 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -5,7 +5,7 @@ name: mellanox-plugin title: Mellanox ConnectX-4 Openstack Features # Plugin version -version: 3.1.8 +version: 3.1.9 # Description description: Enable features over Mellanox ConnectX-4 Adapters diff --git a/pre_build_hook b/pre_build_hook index 848b306..21a8a23 100755 --- a/pre_build_hook +++ b/pre_build_hook @@ -62,7 +62,7 @@ old_debs="${PLUGIN_DIR}/repositories/ubuntu/*.deb" deb_files="cirros-testvm-mellanox_0.3.2-ubuntu3_amd64.deb cirros-testvm-mellanox-ib_0.3.2-9_amd64.deb eswitchd_1.0.0-18_amd64.deb - mlnx-ofed-fuel_3.3-0.1.0.0_amd64.deb + mlnx-ofed-fuel_3.3-0.1.3.1_amd64.deb lldpd_0.9.1-0_amd64.deb python-networking-mlnx_7.0.0-1_all.deb" get_packages "deb" "$old_debs" "$deb_files" diff --git a/scripts/create_mellanox_bootstrap b/scripts/create_mellanox_bootstrap index da1114e..22a6862 100755 --- a/scripts/create_mellanox_bootstrap +++ b/scripts/create_mellanox_bootstrap @@ -22,6 +22,9 @@ import sys import argparse import textwrap +# Load the defaults +DEFAULT_MAX_NUM_VFS = 16 + parser = argparse.ArgumentParser( prog=os.path.basename(sys.argv[0]), formatter_class=argparse.RawDescriptionHelpFormatter, @@ -33,6 +36,8 @@ parser = argparse.ArgumentParser( - current for leaving link type as is ''')) parser.add_argument("--link_type", choices=['eth', 'ib', 'current']) +parser.add_argument("--max_num_vfs", type=int, default = DEFAULT_MAX_NUM_VFS, + help='an integer for the maximum number of vfs to be burned in bootstrap') args = parser.parse_args() if not args.link_type: @@ -44,6 +49,8 @@ else: else: link_type = "_{0}".format(args.link_type) +os.environ['MAX_NUM_VFS'] = '{0}'.format(args.max_num_vfs) + # Set variables plugin = subprocess.Popen("echo /var/www/nailgun/plugins/mellanox-plugin-*/ " + "| tr '/' '\n' | grep mellanox-plugin | tr -d '\n' ",