summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRawan Herzallah <rherzallah@asaltech.com>2016-05-30 12:02:17 +0000
committerRawan Herzallah <rherzallah@asaltech.com>2016-06-08 11:10:18 +0000
commit2dbb3acb3f79beecda6bf072c4dec52f9485570d (patch)
treede82d02f7d8fa00984367eeace371bf49f9f5d31
parenta29bb12057217cad60c2c98d0252cdd8e2fbbfc2 (diff)
Dynamically set maximum number of VFs in bootstrap stage
Allow the user to set the maximum number of vfs burned in bootstrap stage with 16 vfs as default. Change-Id: I0c06a3fa64d196b9c6a3af9aca02560675c6b3b1
Notes
Notes (review): Code-Review+1: Rawan Herzallah <rherzallah@asaltech.com> Code-Review+2: Aviram Bar-Haim <aviramb@mellanox.com> Workflow+1: Aviram Bar-Haim <aviramb@mellanox.com> Verified+2: Jenkins Submitted-by: Jenkins Submitted-at: Thu, 09 Jun 2016 10:55:49 +0000 Reviewed-on: https://review.openstack.org/322795 Project: openstack/fuel-plugin-mellanox Branch: refs/heads/master
-rwxr-xr-xbootstrap/sync/opt/mellanox_init.sh3
-rwxr-xr-xbootstrap/sync/usr/bin/mlnx_fw_upgrade2
-rwxr-xr-xdeployment_scripts/mellanox_settings.py19
-rwxr-xr-xdeployment_scripts/sriov.sh13
-rw-r--r--deployment_tasks.yaml21
-rw-r--r--environment_config.yaml8
-rw-r--r--metadata.yaml2
-rwxr-xr-xpre_build_hook2
-rwxr-xr-xscripts/create_mellanox_bootstrap7
9 files changed, 67 insertions, 10 deletions
diff --git a/bootstrap/sync/opt/mellanox_init.sh b/bootstrap/sync/opt/mellanox_init.sh
index 995520f..597923b 100755
--- a/bootstrap/sync/opt/mellanox_init.sh
+++ b/bootstrap/sync/opt/mellanox_init.sh
@@ -23,6 +23,9 @@ if [ ! -z $1 ]; then
23 sed -i "1a\export LINK_TYPE=$1" /etc/rc.local 23 sed -i "1a\export LINK_TYPE=$1" /etc/rc.local
24fi 24fi
25 25
26# Set MAX_NUM_VFS to run on boot
27sed -i "1a\export MAX_NUM_VFS=$MAX_NUM_VFS" /etc/rc.local
28
26# Install required packages 29# Install required packages
27dpkg -i ${OFED_DEBS_DIR}/mlnx-ofed-kernel-utils*.deb 30dpkg -i ${OFED_DEBS_DIR}/mlnx-ofed-kernel-utils*.deb
28dpkg -i ${OFED_DEBS_DIR}/mlnx-ofed-kernel-dkms*.deb 31dpkg -i ${OFED_DEBS_DIR}/mlnx-ofed-kernel-dkms*.deb
diff --git a/bootstrap/sync/usr/bin/mlnx_fw_upgrade b/bootstrap/sync/usr/bin/mlnx_fw_upgrade
index 8b73a0a..c436db5 100755
--- a/bootstrap/sync/usr/bin/mlnx_fw_upgrade
+++ b/bootstrap/sync/usr/bin/mlnx_fw_upgrade
@@ -15,7 +15,7 @@
15# limitations under the License. 15# limitations under the License.
16 16
17# Set defaults 17# Set defaults
18MAX_VFS=16 18MAX_VFS=$MAX_NUM_VFS
19LOG_FILE=/var/log/mlnx_fw_update.log 19LOG_FILE=/var/log/mlnx_fw_update.log
20OFED_DEBS_DIR=/opt/ofed/MLNX_OFED/DEBS/ 20OFED_DEBS_DIR=/opt/ofed/MLNX_OFED/DEBS/
21 21
diff --git a/deployment_scripts/mellanox_settings.py b/deployment_scripts/mellanox_settings.py
index 658198b..0822c59 100755
--- a/deployment_scripts/mellanox_settings.py
+++ b/deployment_scripts/mellanox_settings.py
@@ -22,6 +22,7 @@ import glob
22import logging 22import logging
23import traceback 23import traceback
24 24
25MAX_NUM_VFS = 16
25MLNX_SECTION = 'mellanox-plugin' 26MLNX_SECTION = 'mellanox-plugin'
26SETTINGS_FILE = '/etc/astute.yaml' 27SETTINGS_FILE = '/etc/astute.yaml'
27PLUGIN_OVERRIDE_FILE = '/etc/hiera/override/plugins.yaml' 28PLUGIN_OVERRIDE_FILE = '/etc/hiera/override/plugins.yaml'
@@ -292,6 +293,22 @@ class MellanoxSettings(object):
292 return cls.get_mlnx_section()['vxlan_offloading'] 293 return cls.get_mlnx_section()['vxlan_offloading']
293 294
294 @classmethod 295 @classmethod
296 def add_reboot_condition(cls):
297 # if MAX_NUM_VF > default which is 16, reboot
298 mlnx = cls.get_mlnx_section()
299 mst_start = os.popen('mst start;').readlines()
300 burned_num_vfs_list = list()
301 devices = os.popen('mst status -v| grep pciconf | grep {0} | awk \'{{print $2}}\' '.format(
302 mlnx['cx_card'].replace("-",""))).readlines()
303 for dev in devices:
304 num = os.popen('mlxconfig -d {0} q | grep NUM_OF_VFS | awk \'{{print $2}}\' \
305 '.format(dev.rsplit()[0])).readlines()
306 burned_num_vfs_list.append(num[0].rsplit()[0])
307 burned_num_vfs = list(set(burned_num_vfs_list))[0]
308 if burned_num_vfs > MAX_NUM_VFS or mlnx['num_of_vfs'] > MAX_NUM_VFS :
309 mlnx['reboot_required'] = True
310
311 @classmethod
295 def update_role_settings(cls): 312 def update_role_settings(cls):
296 # detect ConnectX card 313 # detect ConnectX card
297 cls.add_cx_card() 314 cls.add_cx_card()
@@ -305,6 +322,8 @@ class MellanoxSettings(object):
305 cls.add_storage_vlan() 322 cls.add_storage_vlan()
306 cls.add_iser_interface_name() 323 cls.add_iser_interface_name()
307 cls.set_storage_networking_scheme() 324 cls.set_storage_networking_scheme()
325 # fill reboot condition
326 cls.add_reboot_condition()
308 327
309 @classmethod 328 @classmethod
310 def read_from_yaml(cls, settings_file): 329 def read_from_yaml(cls, settings_file):
diff --git a/deployment_scripts/sriov.sh b/deployment_scripts/sriov.sh
index a79c64b..22048ef 100755
--- a/deployment_scripts/sriov.sh
+++ b/deployment_scripts/sriov.sh
@@ -178,7 +178,18 @@ function burn_vfs_in_fw () {
178 service mst stop &>/dev/null 178 service mst stop &>/dev/null
179 fi 179 fi
180 if [ $CX == 'ConnectX-4' ]; then 180 if [ $CX == 'ConnectX-4' ]; then
181 logger_print debug "Skipping burning ConnectX-4 as it is burnt in bootstrap stage." 181 # required for mlxconfig to discover mlnx devices
182 service openibd start &>/dev/null
183 service mst start &>/dev/null
184 devices=$(mst status -v | grep $CX| grep pciconf | awk '{print $2}')
185 for dev in $devices; do
186 current_fw_vfs=`mlxconfig -d $dev q | grep NUM_OF_VFS | awk '{print $2}'`
187 if [ "$total_vfs" -gt "$current_fw_vfs" ]; then
188 logger_print debug "device=$dev"
189 logger_print debug "Trying mlxconfig -d ${dev} -y set NUM_OF_VFS=${total_vfs}"
190 mlxconfig -d $dev -y set NUM_OF_VFS=$total_vfs
191 fi
192 done
182 fi 193 fi
183} 194}
184 195
diff --git a/deployment_tasks.yaml b/deployment_tasks.yaml
index 86e5fa4..eabb268 100644
--- a/deployment_tasks.yaml
+++ b/deployment_tasks.yaml
@@ -34,7 +34,7 @@
34# this script links astute.yaml to any <role>.yaml on each node. 34# this script links astute.yaml to any <role>.yaml on each node.
35- id: link_astute_file 35- id: link_astute_file
36 role: '*' 36 role: '*'
37 required_for: [customize_mellanox_hiera] 37 required_for: [validate_kernel_devel]
38 requires: [init_mlnx_logs] 38 requires: [init_mlnx_logs]
39 type: shell 39 type: shell
40 parameters: 40 parameters:
@@ -44,7 +44,7 @@
44- id: validate_kernel_devel 44- id: validate_kernel_devel
45 role: '*' 45 role: '*'
46 required_for: [install_mlnx_ofed] 46 required_for: [install_mlnx_ofed]
47 requires: [customize_mellanox_hiera] 47 requires: [link_astute_file]
48 type: puppet 48 type: puppet
49 parameters: 49 parameters:
50 puppet_manifest: puppet/manifests/verify_kernel_devel.pp 50 puppet_manifest: puppet/manifests/verify_kernel_devel.pp
@@ -53,7 +53,7 @@
53# Install OFED + FW upgrade 53# Install OFED + FW upgrade
54- id: install_mlnx_ofed 54- id: install_mlnx_ofed
55 role: '*' 55 role: '*'
56 required_for: [customize_mellanox_hiera_post] 56 required_for: [customize_mellanox_hiera]
57 requires: [validate_kernel_devel] 57 requires: [validate_kernel_devel]
58 type: shell 58 type: shell
59 parameters: 59 parameters:
@@ -61,7 +61,7 @@
61 timeout: 5000 61 timeout: 5000
62# Add relevant settings for Mellanox manifests to mellanox plugin section in 62# Add relevant settings for Mellanox manifests to mellanox plugin section in
63# Hiera, to make the data easily accessible and independent of astute.yaml 63# Hiera, to make the data easily accessible and independent of astute.yaml
64- id: customize_mellanox_hiera_post 64- id: customize_mellanox_hiera
65 role: '*' 65 role: '*'
66 required_for: [configure_vxlan_offload] 66 required_for: [configure_vxlan_offload]
67 requires: [install_mlnx_ofed] 67 requires: [install_mlnx_ofed]
@@ -73,7 +73,7 @@
73- id: configure_vxlan_offload 73- id: configure_vxlan_offload
74 role: '*' 74 role: '*'
75 required_for: [configure_sriov] 75 required_for: [configure_sriov]
76 requires: [customize_mellanox_hiera_post] 76 requires: [customize_mellanox_hiera]
77 type: shell 77 type: shell
78 parameters: 78 parameters:
79 cmd: ./vxlan_offloading.sh configure 79 cmd: ./vxlan_offloading.sh configure
@@ -91,13 +91,22 @@
91# if VXLAN is not required. Rename iSER interface for Eth mode 91# if VXLAN is not required. Rename iSER interface for Eth mode
92- id: rename_iser_probe_vf 92- id: rename_iser_probe_vf
93 role: '*' 93 role: '*'
94 required_for: [validate_sriov] 94 required_for: [reboot_after_max_vfs_num_change]
95 requires: [configure_sriov] 95 requires: [configure_sriov]
96 type: puppet 96 type: puppet
97 parameters: 97 parameters:
98 puppet_manifest: puppet/manifests/iser_rename.pp 98 puppet_manifest: puppet/manifests/iser_rename.pp
99 puppet_modules: puppet/modules:/etc/puppet/modules 99 puppet_modules: puppet/modules:/etc/puppet/modules
100 timeout: 300 100 timeout: 300
101# Reboot due to changing MAX number of VFs
102- id: reboot_after_max_vfs_num_change
103 role: '*'
104 required_for: [validate_sriov]
105 requires: [rename_iser_probe_vf]
106 type: reboot
107 condition: "settings:mellanox-plugin.reboot_required.value == true"
108 parameters:
109 timeout: 6000
101# Check number of VFs 110# Check number of VFs
102- id: validate_sriov 111- id: validate_sriov
103 role: '*' 112 role: '*'
diff --git a/environment_config.yaml b/environment_config.yaml
index af90a3f..b70a9a0 100644
--- a/environment_config.yaml
+++ b/environment_config.yaml
@@ -26,6 +26,14 @@ attributes:
26 - condition: "settings:mellanox-plugin.sriov.value == false" 26 - condition: "settings:mellanox-plugin.sriov.value == false"
27 message: "In order to change the number of virtual NICs, Neutron SR-IOV plugin should be checked." 27 message: "In order to change the number of virtual NICs, Neutron SR-IOV plugin should be checked."
28 28
29 reboot_required:
30 value: false
31 label: "Reboot when maximum number of vfs configured by the user is greater than the one burned in FW."
32 description: |
33 Reboots Slave nodes if the maximum number of vfs configured by the user is greator than the number burned in FW.
34 weight: 30
35 type: hidden
36
29 mlnx_qos: 37 mlnx_qos:
30 value: false 38 value: false
31 label: "Quality of Service over Mellanox SR-IOV ports (Neutron)" 39 label: "Quality of Service over Mellanox SR-IOV ports (Neutron)"
diff --git a/metadata.yaml b/metadata.yaml
index b8ba0ca..df05e57 100644
--- a/metadata.yaml
+++ b/metadata.yaml
@@ -5,7 +5,7 @@ name: mellanox-plugin
5title: Mellanox ConnectX-4 Openstack Features 5title: Mellanox ConnectX-4 Openstack Features
6 6
7# Plugin version 7# Plugin version
8version: 3.1.8 8version: 3.1.9
9 9
10# Description 10# Description
11description: Enable features over Mellanox ConnectX-4 Adapters 11description: Enable features over Mellanox ConnectX-4 Adapters
diff --git a/pre_build_hook b/pre_build_hook
index 848b306..21a8a23 100755
--- a/pre_build_hook
+++ b/pre_build_hook
@@ -62,7 +62,7 @@ old_debs="${PLUGIN_DIR}/repositories/ubuntu/*.deb"
62deb_files="cirros-testvm-mellanox_0.3.2-ubuntu3_amd64.deb 62deb_files="cirros-testvm-mellanox_0.3.2-ubuntu3_amd64.deb
63 cirros-testvm-mellanox-ib_0.3.2-9_amd64.deb 63 cirros-testvm-mellanox-ib_0.3.2-9_amd64.deb
64 eswitchd_1.0.0-18_amd64.deb 64 eswitchd_1.0.0-18_amd64.deb
65 mlnx-ofed-fuel_3.3-0.1.0.0_amd64.deb 65 mlnx-ofed-fuel_3.3-0.1.3.1_amd64.deb
66 lldpd_0.9.1-0_amd64.deb 66 lldpd_0.9.1-0_amd64.deb
67 python-networking-mlnx_7.0.0-1_all.deb" 67 python-networking-mlnx_7.0.0-1_all.deb"
68get_packages "deb" "$old_debs" "$deb_files" 68get_packages "deb" "$old_debs" "$deb_files"
diff --git a/scripts/create_mellanox_bootstrap b/scripts/create_mellanox_bootstrap
index da1114e..22a6862 100755
--- a/scripts/create_mellanox_bootstrap
+++ b/scripts/create_mellanox_bootstrap
@@ -22,6 +22,9 @@ import sys
22import argparse 22import argparse
23import textwrap 23import textwrap
24 24
25# Load the defaults
26DEFAULT_MAX_NUM_VFS = 16
27
25parser = argparse.ArgumentParser( 28parser = argparse.ArgumentParser(
26 prog=os.path.basename(sys.argv[0]), 29 prog=os.path.basename(sys.argv[0]),
27 formatter_class=argparse.RawDescriptionHelpFormatter, 30 formatter_class=argparse.RawDescriptionHelpFormatter,
@@ -33,6 +36,8 @@ parser = argparse.ArgumentParser(
33 - current for leaving link type as is 36 - current for leaving link type as is
34 ''')) 37 '''))
35parser.add_argument("--link_type", choices=['eth', 'ib', 'current']) 38parser.add_argument("--link_type", choices=['eth', 'ib', 'current'])
39parser.add_argument("--max_num_vfs", type=int, default = DEFAULT_MAX_NUM_VFS,
40 help='an integer for the maximum number of vfs to be burned in bootstrap')
36 41
37args = parser.parse_args() 42args = parser.parse_args()
38if not args.link_type: 43if not args.link_type:
@@ -44,6 +49,8 @@ else:
44 else: 49 else:
45 link_type = "_{0}".format(args.link_type) 50 link_type = "_{0}".format(args.link_type)
46 51
52os.environ['MAX_NUM_VFS'] = '{0}'.format(args.max_num_vfs)
53
47# Set variables 54# Set variables
48plugin = subprocess.Popen("echo /var/www/nailgun/plugins/mellanox-plugin-*/ " + 55plugin = subprocess.Popen("echo /var/www/nailgun/plugins/mellanox-plugin-*/ " +
49 "| tr '/' '\n' | grep mellanox-plugin | tr -d '\n' ", 56 "| tr '/' '\n' | grep mellanox-plugin | tr -d '\n' ",