summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRawan Herzallah <rherzallah@asaltech.com>2016-04-04 14:24:01 +0000
committerRawan Herzallah <rherzallah@asaltech.com>2016-04-21 14:06:07 +0000
commit42c47329fcc644093ef91686b9ce3000b232a127 (patch)
tree4ac548746475923ba6e38164e036e7bfbd2c4e50
parent8f7a03adf558d40acd0a4fc97775f4bb20ede0f5 (diff)
Implement ConnectX-4 configurations
Add changes to implement features for ConnectX-4 adapters Change-Id: Ie66162a2cb37ea3c1766f20bf218fb239eca8cf3
Notes
Notes (review): Code-Review+1: Rawan Herzallah <rherzallah@asaltech.com> Code-Review+2: Aviram Bar-Haim <aviramb@mellanox.com> Workflow+1: Aviram Bar-Haim <aviramb@mellanox.com> Verified+2: Jenkins Submitted-by: Jenkins Submitted-at: Thu, 21 Apr 2016 15:53:20 +0000 Reviewed-on: https://review.openstack.org/301159 Project: openstack/fuel-plugin-mellanox Branch: refs/heads/master
-rwxr-xr-xdeployment_scripts/common4
-rwxr-xr-xdeployment_scripts/configure_qos.sh7
-rwxr-xr-xdeployment_scripts/install_ofed.sh41
-rwxr-xr-xdeployment_scripts/mellanox_settings.py79
-rwxr-xr-xdeployment_scripts/post_set_mellanox_interfaces.sh32
-rw-r--r--deployment_scripts/puppet/manifests/compute.pp5
-rw-r--r--deployment_scripts/puppet/manifests/controller.pp3
-rw-r--r--deployment_scripts/puppet/manifests/iser_rename.pp2
-rw-r--r--deployment_scripts/puppet/modules/mellanox_openstack/manifests/compute_sriov.pp3
-rw-r--r--deployment_scripts/puppet/modules/mellanox_openstack/manifests/controller_sriov.pp3
-rw-r--r--deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb11
-rwxr-xr-xdeployment_scripts/sriov.sh116
-rwxr-xr-xdeployment_scripts/vxlan_offloading.sh14
-rw-r--r--deployment_tasks.yaml34
-rw-r--r--environment_config.yaml2
-rw-r--r--metadata.yaml13
-rwxr-xr-xpre_build_hook2
17 files changed, 242 insertions, 129 deletions
diff --git a/deployment_scripts/common b/deployment_scripts/common
index 3c1f61a..a2bf050 100755
--- a/deployment_scripts/common
+++ b/deployment_scripts/common
@@ -86,7 +86,9 @@ readonly USER_NUM_OF_VFS=`get_mlnx_param num_of_vfs`
86readonly ISER=`get_mlnx_param iser` 86readonly ISER=`get_mlnx_param iser`
87readonly MAX_VFS=62 87readonly MAX_VFS=62
88readonly MIN_VFS=1 88readonly MIN_VFS=1
89readonly CX='ConnectX3' 89readonly CX=`get_mlnx_param cx_card`
90readonly NETWORK_TYPE=`get_mlnx_param network_type`
91readonly PHYSICAL_PORT=`get_mlnx_param physical_port`
90readonly VXLAN_OFFLOADING=`get_mlnx_param vxlan_offloading` 92readonly VXLAN_OFFLOADING=`get_mlnx_param vxlan_offloading`
91readonly ROLES=`get_param roles` 93readonly ROLES=`get_param roles`
92readonly ROLE=`get_param role` 94readonly ROLE=`get_param role`
diff --git a/deployment_scripts/configure_qos.sh b/deployment_scripts/configure_qos.sh
index 2750726..8180c08 100755
--- a/deployment_scripts/configure_qos.sh
+++ b/deployment_scripts/configure_qos.sh
@@ -64,7 +64,12 @@ function configure_qos () {
64 64
65case $SCRIPT_MODE in 65case $SCRIPT_MODE in
66 'configure') 66 'configure')
67 configure_qos 67 if [ "$CX" == "ConnectX-3" ]; then
68 configure_qos
69 fi
70 if [ "$CX" == "ConnectX-4" ]; then
71 logger_print info "QoS is not implemented for ConnectX-4."
72 fi
68 ;; 73 ;;
69 *) 74 *)
70 logger_print error "Unsupported execution mode ${SCRIPT_MODE}." 75 logger_print error "Unsupported execution mode ${SCRIPT_MODE}."
diff --git a/deployment_scripts/install_ofed.sh b/deployment_scripts/install_ofed.sh
index 7b23082..df7da01 100755
--- a/deployment_scripts/install_ofed.sh
+++ b/deployment_scripts/install_ofed.sh
@@ -87,7 +87,8 @@ function install_ofed_without_fw_update () {
87 87
88 logger_print info "Installing OFED drivers" 88 logger_print info "Installing OFED drivers"
89 OFED_INSTALL_SCRIPT_CMD="/usr/bin/perl ${OFED_INSTALL_SCRIPT}" 89 OFED_INSTALL_SCRIPT_CMD="/usr/bin/perl ${OFED_INSTALL_SCRIPT}"
90 ${OFED_INSTALL_SCRIPT_CMD} --force --enable-sriov --without-fw-update 90 ${OFED_INSTALL_SCRIPT_CMD} --force --without-fw-update
91
91 rc=$? 92 rc=$?
92 if [ $rc -ne 0 ]; then 93 if [ $rc -ne 0 ]; then
93 logger_print error "Failed execute ${OFED_INSTALL_SCRIPT_CMD} error code ${rc}" 94 logger_print error "Failed execute ${OFED_INSTALL_SCRIPT_CMD} error code ${rc}"
@@ -97,36 +98,8 @@ function install_ofed_without_fw_update () {
97 fi 98 fi
98} 99}
99 100
100function update_fw_if_not_oem () {
101 BUS_ID=`lspci | grep -m 1 Mellanox | cut -d' ' -f1`
102 if [ -z $BUS_ID ]; then
103 logger_print info "Didn't find bus, skipping firmware upgrade"
104 exit 0
105 fi
106
107 mstflint -d ${BUS_ID} q | grep -i PSID | grep MT_
108 if [ $? -ne 0 ]; then
109 logger_print info "Not Mellanox Card, skipping firmware upgrade"
110 exit 0
111 fi
112
113 OFED_INSTALL_SCRIPT="${OFED_DIR}/mlnxofedinstall"
114 if [ ! -f $OFED_INSTALL_SCRIPT ]; then
115 logger_print error "Failed to find $OFED_INSTALL_SCRIPT"
116 exit 1
117 fi
118
119 logger_print info "Updating FW on Mellanox HCA with BUS ID = ${BUS_ID}"
120
121 OFED_INSTALL_SCRIPT_CMD="/usr/bin/perl ${OFED_INSTALL_SCRIPT}"
122 ${OFED_INSTALL_SCRIPT_CMD} --force --enable-sriov --fw-update-only
123 if [ $? -ne 0 ]; then
124 logger_print error "Failed execute ${OFED_INSTALL_SCRIPT_CMD} error code $?"
125 fi
126}
127
128function enable_eipoib (){ 101function enable_eipoib (){
129 if [ $DRIVER == 'eth_ipoib' ]; then 102 if [ $NETWORK_TYPE == 'infiniband' ]; then
130 sed -i s/^E_IPOIB_LOAD.*$/E_IPOIB_LOAD=yes/g /etc/infiniband/openib.conf 103 sed -i s/^E_IPOIB_LOAD.*$/E_IPOIB_LOAD=yes/g /etc/infiniband/openib.conf
131 104
132 # Set Buffers size 105 # Set Buffers size
@@ -161,9 +134,9 @@ if ! is_ofed_installed; then
161 enable_eipoib 134 enable_eipoib
162fi 135fi
163 136
164# OEM cards require a different dedicated OFED build, this build doesn't
165# support them.
166update_fw_if_not_oem
167
168# Decrease loglevels for prevent flooding kernel messages to console 137# Decrease loglevels for prevent flooding kernel messages to console
169sysctl_conf set 'kernel.printk' '4 4 1 7' 138sysctl_conf set 'kernel.printk' '4 4 1 7'
139service openibd stop
140service openibd start
141
142exit 0
diff --git a/deployment_scripts/mellanox_settings.py b/deployment_scripts/mellanox_settings.py
index 6219bdd..9b4a034 100755
--- a/deployment_scripts/mellanox_settings.py
+++ b/deployment_scripts/mellanox_settings.py
@@ -16,6 +16,7 @@
16 16
17import os 17import os
18import sys 18import sys
19import subprocess
19import yaml 20import yaml
20import glob 21import glob
21import logging 22import logging
@@ -24,7 +25,16 @@ import traceback
24MLNX_SECTION = 'mellanox-plugin' 25MLNX_SECTION = 'mellanox-plugin'
25SETTINGS_FILE = '/etc/astute.yaml' 26SETTINGS_FILE = '/etc/astute.yaml'
26PLUGIN_OVERRIDE_FILE = '/etc/hiera/override/plugins.yaml' 27PLUGIN_OVERRIDE_FILE = '/etc/hiera/override/plugins.yaml'
27MLNX_DRIVERS_LIST = ('mlx4_en', 'eth_ipoib') 28MLNX_DRIVERS_LIST = { 'ConnectX-3': {'eth_driver' : 'mlx4_en', 'ib_driver' : 'eth_ipoib'},
29 'ConnectX-4': {'eth_driver' : 'mlx5_core', 'ib_driver' : 'eth_ipoib'}}
30MLNX_DRIVERS = set([MLNX_DRIVERS_LIST[card][net]
31 for card in MLNX_DRIVERS_LIST
32 for net in MLNX_DRIVERS_LIST[card]])
33ETH_DRIVERS = set([MLNX_DRIVERS_LIST[card][net]
34 for card in MLNX_DRIVERS_LIST
35 for net in MLNX_DRIVERS_LIST[card]
36 if net == 'eth_driver'])
37IB_DRIVERS = MLNX_DRIVERS - ETH_DRIVERS
28ISER_IFC_NAME = 'mlnx_iser0' 38ISER_IFC_NAME = 'mlnx_iser0'
29LOG_FILE = '/var/log/mellanox-plugin.log' 39LOG_FILE = '/var/log/mellanox-plugin.log'
30 40
@@ -64,6 +74,56 @@ class MellanoxSettings(object):
64 return ifc 74 return ifc
65 75
66 @classmethod 76 @classmethod
77 def get_card_type(cls, driver):
78 for card in MLNX_DRIVERS_LIST.keys():
79 if driver in MLNX_DRIVERS_LIST[card].values():
80 network_driver_type = MLNX_DRIVERS_LIST[card].keys()[MLNX_DRIVERS_LIST[card].values()\
81 .index(driver)]
82 return card
83
84 @classmethod
85 def add_cx_card(cls):
86 mlnx_interfaces = cls.mlnx_interfaces_section
87 drivers = list()
88 interfaces = list()
89 mlnx = cls.get_mlnx_section()
90 for network_type, ifc_dict in mlnx_interfaces.iteritems():
91 if 'driver' in ifc_dict and network_type in ['private','management','storage']:
92 drivers.append(ifc_dict['driver'])
93 interfaces.append(ifc_dict['interface'])
94
95 drivers_set = list(set(drivers))
96 interfaces_set = list(set(interfaces))
97 if (len(drivers_set) > 1):
98 logging.error("Multiple ConnectX adapters was found in this environment.")
99 raise MellanoxSettingsException(
100 "Multiple ConnectX adapters was found in this environment."
101 )
102 else:
103 current_driver = drivers_set[0]
104 mellanox_interface = interfaces_set[0]
105 if current_driver in ETH_DRIVERS:
106 mlnx['network_type'] = 'ethernet'
107 mlnx['cx_card'] = cls.get_card_type(current_driver)
108 elif current_driver in IB_DRIVERS:
109 mlnx['network_type'] = 'infiniband'
110 ibdev = os.popen('ibdev2netdev').readlines()
111 if not ibdev:
112 mlnx['cx_card'] = 'none'
113 logging.error('Failed executing ibdev2netdev')
114 return 0
115 interface_line = [l for l in ibdev if mellanox_interface in l]
116 if interface_line and 'mlx5' in interface_line.pop():
117 mlnx['cx_card'] = 'ConnectX-4'
118 else:
119 mlnx['cx_card'] = 'ConnectX-3'
120
121 network_info_msg = 'Detected Network Type is: %s ', mlnx['network_type']
122 card_info_msg = 'Detected Card Type is: %s ', mlnx['cx_card']
123 logging.info(network_info_msg)
124 logging.info(card_info_msg)
125
126 @classmethod
67 def add_driver(cls): 127 def add_driver(cls):
68 interfaces = cls.get_interfaces_section() 128 interfaces = cls.get_interfaces_section()
69 mlnx = cls.get_mlnx_section() 129 mlnx = cls.get_mlnx_section()
@@ -87,14 +147,14 @@ class MellanoxSettings(object):
87 mlnx = cls.get_mlnx_section() 147 mlnx = cls.get_mlnx_section()
88 148
89 private_ifc = cls.get_interface_by_network('private') 149 private_ifc = cls.get_interface_by_network('private')
90 if mlnx['driver'] == 'eth_ipoib': 150 if mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['ib_driver']:
91 if 'bus_info' not in interfaces[private_ifc]['vendor_specific']: 151 if 'bus_info' not in interfaces[private_ifc]['vendor_specific']:
92 raise MellanoxSettingsException( 152 raise MellanoxSettingsException(
93 "Couldn't find 'bus_info' for interface " 153 "Couldn't find 'bus_info' for interface "
94 "{0}".format(private_ifc) 154 "{0}".format(private_ifc)
95 ) 155 )
96 mlnx['physical_port'] = interfaces[private_ifc]['vendor_specific']['bus_info'] 156 mlnx['physical_port'] = interfaces[private_ifc]['vendor_specific']['bus_info']
97 elif mlnx['driver'] == 'mlx4_en': 157 elif mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['eth_driver']:
98 mlnx['physical_port'] = private_ifc 158 mlnx['physical_port'] = private_ifc
99 159
100 @classmethod 160 @classmethod
@@ -110,7 +170,7 @@ class MellanoxSettings(object):
110 raise MellanoxSettingsException( 170 raise MellanoxSettingsException(
111 "Failed reading vlan for br-storage" 171 "Failed reading vlan for br-storage"
112 ) 172 )
113 if mlnx['driver'] == 'eth_ipoib': 173 if mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['ib_driver']:
114 pkey = format((int(vlan) ^ 0x8000),'04x') 174 pkey = format((int(vlan) ^ 0x8000),'04x')
115 mlnx['storage_pkey'] = pkey 175 mlnx['storage_pkey'] = pkey
116 176
@@ -124,9 +184,9 @@ class MellanoxSettings(object):
124 def add_iser_interface_name(cls): 184 def add_iser_interface_name(cls):
125 mlnx = cls.get_mlnx_section() 185 mlnx = cls.get_mlnx_section()
126 storage_ifc = cls.get_interface_by_network('storage') 186 storage_ifc = cls.get_interface_by_network('storage')
127 if mlnx['driver'] == 'mlx4_en': 187 if mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['eth_driver']:
128 mlnx['iser_ifc_name'] = ISER_IFC_NAME 188 mlnx['iser_ifc_name'] = ISER_IFC_NAME
129 elif mlnx['driver'] == 'eth_ipoib': 189 elif mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['ib_driver']:
130 interfaces = cls.get_interfaces_section() 190 interfaces = cls.get_interfaces_section()
131 mlnx['iser_ifc_name'] = interfaces[storage_ifc]['vendor_specific']['bus_info'] 191 mlnx['iser_ifc_name'] = interfaces[storage_ifc]['vendor_specific']['bus_info']
132 else: 192 else:
@@ -150,7 +210,7 @@ class MellanoxSettings(object):
150 # Handle iSER interface with and w/o vlan tagging 210 # Handle iSER interface with and w/o vlan tagging
151 storage_vlan = mlnx.get('storage_vlan') 211 storage_vlan = mlnx.get('storage_vlan')
152 storage_parent = cls.get_interface_by_network('storage') 212 storage_parent = cls.get_interface_by_network('storage')
153 if storage_vlan and mlnx['driver'] == 'mlx4_en': # Use VLAN dev 213 if storage_vlan and mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['eth_driver']: # Use VLAN dev
154 vlan_name = "{0}.{1}".format(ISER_IFC_NAME, storage_vlan) 214 vlan_name = "{0}.{1}".format(ISER_IFC_NAME, storage_vlan)
155 # Set storage rule to iSER interface vlan interface 215 # Set storage rule to iSER interface vlan interface
156 cls.data['network_scheme']['roles']['storage'] = vlan_name 216 cls.data['network_scheme']['roles']['storage'] = vlan_name
@@ -212,9 +272,10 @@ class MellanoxSettings(object):
212 # the main change will be here because it reads phy_interfaces 272 # the main change will be here because it reads phy_interfaces
213 mlnx_interfaces = cls.mlnx_interfaces_section 273 mlnx_interfaces = cls.mlnx_interfaces_section
214 drivers = list() 274 drivers = list()
275 mlnx = cls.get_mlnx_section()
215 for network_type, ifc_dict in mlnx_interfaces.iteritems(): 276 for network_type, ifc_dict in mlnx_interfaces.iteritems():
216 if 'driver' in ifc_dict and \ 277 if 'driver' in ifc_dict and \
217 ifc_dict['driver'] in MLNX_DRIVERS_LIST: 278 ifc_dict['driver'] in MLNX_DRIVERS_LIST[mlnx['cx_card']].values():
218 drivers.append(ifc_dict['driver']) 279 drivers.append(ifc_dict['driver'])
219 return list(set(drivers)) 280 return list(set(drivers))
220 281
@@ -232,6 +293,8 @@ class MellanoxSettings(object):
232 293
233 @classmethod 294 @classmethod
234 def update_role_settings(cls): 295 def update_role_settings(cls):
296 # detect ConnectX card
297 cls.add_cx_card()
235 # realize the driver in use (eth/ib) 298 # realize the driver in use (eth/ib)
236 cls.add_driver() 299 cls.add_driver()
237 # decide the physical function for SR-IOV 300 # decide the physical function for SR-IOV
diff --git a/deployment_scripts/post_set_mellanox_interfaces.sh b/deployment_scripts/post_set_mellanox_interfaces.sh
index b208922..88e72e8 100755
--- a/deployment_scripts/post_set_mellanox_interfaces.sh
+++ b/deployment_scripts/post_set_mellanox_interfaces.sh
@@ -21,22 +21,28 @@ source ./common
21if ([[ $ROLES == *compute* ]] && [[ ! $ROLES == "compute" ]]) \ 21if ([[ $ROLES == *compute* ]] && [[ ! $ROLES == "compute" ]]) \
22 && [ $SRIOV == true ] ; then 22 && [ $SRIOV == true ] ; then
23 23
24 # Update VFs 24 if [ $CX == 'ConnectX-3' ]; then
25 ./sriov.sh configure
26
27 # Kill tgt daemons if exists
28 tgt_locks=`find /var/run/ -name tgtd* | wc -l`
29 if [ $tgt_locks -ne 0 ];then
30 \rm -f /var/run/tgtd* && killall -9 tgtd
31 service tgt stop
32 fi
33 25
34 service openibd restart && service openvswitch-switch restart 26 # Update VFs
27 ./sriov.sh configure
35 28
36 if [ $tgt_locks -ne 0 ];then 29 # Kill tgt daemons if exists
37 service tgt start 30 tgt_locks=`find /var/run/ -name tgtd* | wc -l`
38 fi 31 if [ $tgt_locks -ne 0 ];then
32 \rm -f /var/run/tgtd* && killall -9 tgtd
33 service tgt stop
34 fi
35
36 service openibd restart && service openvswitch-switch restart
39 37
38 if [ $tgt_locks -ne 0 ];then
39 service tgt start
40 fi
41 fi
42 if [ $CX == 'ConnectX-4' ]; then
43 service openibd restart && service openvswitch-switch restart
44 #sleep 10
45 fi
40 # Verify VFs 46 # Verify VFs
41 ./sriov.sh validate 47 ./sriov.sh validate
42 48
diff --git a/deployment_scripts/puppet/manifests/compute.pp b/deployment_scripts/puppet/manifests/compute.pp
index 3c2e229..663e097 100644
--- a/deployment_scripts/puppet/manifests/compute.pp
+++ b/deployment_scripts/puppet/manifests/compute.pp
@@ -5,7 +5,7 @@ $firewall_driver = 'neutron.agent.firewall.NoopFirewallDriver'
5$private_net = $quantum_settings['default_private_net'] 5$private_net = $quantum_settings['default_private_net']
6$roles = hiera('roles') 6$roles = hiera('roles')
7 7
8if ( $mlnx_driver == 'mlx4_en' ){ 8if ( $mlnx['network_type'] == 'ethernet' and $mlnx['iser'] ){
9 $exclude_vf = '0' 9 $exclude_vf = '0'
10} else { 10} else {
11 $exclude_vf = '' 11 $exclude_vf = ''
@@ -16,12 +16,13 @@ if ($mlnx['sriov']) {
16 physnet => $quantum_settings['predefined_networks'][$private_net]['L2']['physnet'], 16 physnet => $quantum_settings['predefined_networks'][$private_net]['L2']['physnet'],
17 physifc => $mlnx['physical_port'], 17 physifc => $mlnx['physical_port'],
18 mlnx_driver => $mlnx['driver'], 18 mlnx_driver => $mlnx['driver'],
19 network_type => $mlnx['network_type'],
19 firewall_driver => $firewall_driver, 20 firewall_driver => $firewall_driver,
20 exclude_vf => $exclude_vf, 21 exclude_vf => $exclude_vf,
21 } 22 }
22} 23}
23 24
24# Configure QoS for ETH 25# Configure QoS for ConnectX3 ETH
25if ( $mlnx['driver'] == 'mlx4_en' and $mlnx['mlnx_qos'] ) { 26if ( $mlnx['driver'] == 'mlx4_en' and $mlnx['mlnx_qos'] ) {
26 class { 'mellanox_openstack::configure_qos' : 27 class { 'mellanox_openstack::configure_qos' :
27 mlnx_sriov => $mlnx['sriov'], 28 mlnx_sriov => $mlnx['sriov'],
diff --git a/deployment_scripts/puppet/manifests/controller.pp b/deployment_scripts/puppet/manifests/controller.pp
index 08fbdd2..a019107 100644
--- a/deployment_scripts/puppet/manifests/controller.pp
+++ b/deployment_scripts/puppet/manifests/controller.pp
@@ -4,7 +4,7 @@ $eswitch_apply_profile_patch = 'True'
4$mechanism_drivers = 'openvswitch' 4$mechanism_drivers = 'openvswitch'
5$roles = hiera('roles') 5$roles = hiera('roles')
6 6
7# Configure QoS for ETH 7# Configure QoS for connectX3 ETH
8if ( $mlnx['driver'] == 'mlx4_en' and $mlnx['mlnx_qos'] ) { 8if ( $mlnx['driver'] == 'mlx4_en' and $mlnx['mlnx_qos'] ) {
9 class { 'mellanox_openstack::configure_qos' : 9 class { 'mellanox_openstack::configure_qos' :
10 mlnx_sriov => $mlnx['sriov'], 10 mlnx_sriov => $mlnx['sriov'],
@@ -20,6 +20,7 @@ if ($mlnx['sriov']) {
20 eswitch_apply_profile_patch => $eswitch_apply_profile_patch, 20 eswitch_apply_profile_patch => $eswitch_apply_profile_patch,
21 mechanism_drivers => $mechanism_drivers, 21 mechanism_drivers => $mechanism_drivers,
22 mlnx_driver => $mlnx['driver'], 22 mlnx_driver => $mlnx['driver'],
23 network_type => $mlnx['network_type'],
23 mlnx_sriov => $mlnx['sriov'], 24 mlnx_sriov => $mlnx['sriov'],
24 pci_vendor_devices => $pci_vendor_devices, 25 pci_vendor_devices => $pci_vendor_devices,
25 agent_required => $agent_required, 26 agent_required => $agent_required,
diff --git a/deployment_scripts/puppet/manifests/iser_rename.pp b/deployment_scripts/puppet/manifests/iser_rename.pp
index ddd9149..3e5b0a9 100644
--- a/deployment_scripts/puppet/manifests/iser_rename.pp
+++ b/deployment_scripts/puppet/manifests/iser_rename.pp
@@ -1,6 +1,6 @@
1$mlnx = hiera('mellanox-plugin') 1$mlnx = hiera('mellanox-plugin')
2 2
3if ($mlnx['iser'] and $mlnx['driver'] == 'mlx4_en') { 3if ($mlnx['iser'] and $mlnx['network_type'] == 'ethernet') {
4 class { 'mellanox_openstack::iser_rename' : 4 class { 'mellanox_openstack::iser_rename' :
5 storage_parent => $mlnx['storage_parent'], 5 storage_parent => $mlnx['storage_parent'],
6 iser_interface_name => $mlnx['iser_ifc_name'], 6 iser_interface_name => $mlnx['iser_ifc_name'],
diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/manifests/compute_sriov.pp b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/compute_sriov.pp
index 9e6dfd2..0c49145 100644
--- a/deployment_scripts/puppet/modules/mellanox_openstack/manifests/compute_sriov.pp
+++ b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/compute_sriov.pp
@@ -2,6 +2,7 @@ class mellanox_openstack::compute_sriov (
2 $physnet, 2 $physnet,
3 $physifc, 3 $physifc,
4 $mlnx_driver, 4 $mlnx_driver,
5 $network_type,
5 $firewall_driver, 6 $firewall_driver,
6 $exclude_vf, 7 $exclude_vf,
7) { 8) {
@@ -35,7 +36,7 @@ class mellanox_openstack::compute_sriov (
35 value => $firewall_driver, 36 value => $firewall_driver,
36 } 37 }
37 38
38 if ( $mlnx_driver == 'mlx4_en' ){ 39 if ( $network_type == 'ethernet' ){
39 package { $sriov_agent_package: 40 package { $sriov_agent_package:
40 ensure => installed, 41 ensure => installed,
41 } 42 }
diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/manifests/controller_sriov.pp b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/controller_sriov.pp
index 2d6afa0..fceb7bc 100644
--- a/deployment_scripts/puppet/modules/mellanox_openstack/manifests/controller_sriov.pp
+++ b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/controller_sriov.pp
@@ -3,6 +3,7 @@ class mellanox_openstack::controller_sriov (
3 $eswitch_apply_profile_patch, 3 $eswitch_apply_profile_patch,
4 $mechanism_drivers, 4 $mechanism_drivers,
5 $mlnx_driver, 5 $mlnx_driver,
6 $network_type,
6 $mlnx_sriov, 7 $mlnx_sriov,
7 $pci_vendor_devices, 8 $pci_vendor_devices,
8 $agent_required, 9 $agent_required,
@@ -30,7 +31,7 @@ class mellanox_openstack::controller_sriov (
30 $sdn_extra_mechanism_driver='' 31 $sdn_extra_mechanism_driver=''
31 } 32 }
32 33
33 if ( $mlnx_driver == 'mlx4_en' ){ 34 if ( $network_type == 'ethernet' ){
34 $ml2_extra_mechanism_driver = "${sdn_extra_mechanism_driver}sriovnicswitch" 35 $ml2_extra_mechanism_driver = "${sdn_extra_mechanism_driver}sriovnicswitch"
35 neutron_plugin_ml2 { 36 neutron_plugin_ml2 {
36 'ml2/mechanism_drivers': value => "${ml2_extra_mechanism_driver},${mechanism_drivers}"; 37 'ml2/mechanism_drivers': value => "${ml2_extra_mechanism_driver},${mechanism_drivers}";
diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb b/deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb
index 9bc9ac8..c957ea2 100644
--- a/deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb
+++ b/deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb
@@ -93,7 +93,16 @@ if [ $PROBED_PORT_NAME != $ISER_NAME ]; then
93 fi 93 fi
94 94
95 # restart OFED modules for udev changes to take effect 95 # restart OFED modules for udev changes to take effect
96 modprobe -r mlx4_en && modprobe mlx4_en 96 modprobe -r mlx4_en
97 modprobe -r mlx5_ib && modprobe -r mlx5_core
98 udevadm control --reload-rules
99 udevadm trigger
100 modprobe mlx4_en
101 modprobe mlx5_ib && modprobe mlx5_core
102 echo "#!/bin/bash" > /etc/network/if-up.d/iser_ifc
103 echo "ifconfig $ISER_NAME up " >> /etc/network/if-up.d/iser_ifc
104 ifup --all
105
97 if [ $? -ne 0 ]; then 106 if [ $? -ne 0 ]; then
98 logger_print error "Mellanox drivers restart failed." 107 logger_print error "Mellanox drivers restart failed."
99 exit 1 108 exit 1
diff --git a/deployment_scripts/sriov.sh b/deployment_scripts/sriov.sh
index 347ca2f..d5787f5 100755
--- a/deployment_scripts/sriov.sh
+++ b/deployment_scripts/sriov.sh
@@ -26,16 +26,16 @@ readonly GRUB_FILE_CENTOS="/boot/grub/grub.conf"
26readonly GRUB_FILE_UBUNTU="/boot/grub/grub.cfg" 26readonly GRUB_FILE_UBUNTU="/boot/grub/grub.cfg"
27 27
28function get_port_type() { 28function get_port_type() {
29 if [ $DRIVER == 'mlx4_en' ]; then 29 if [ $NETWORK_TYPE == 'ethernet' ]; then
30 port_type=2 30 port_type=2
31 elif [ $DRIVER == 'eth_ipoib' ]; then 31 else
32 port_type=1 32 port_type=1
33 fi 33 fi
34 echo $port_type 34 echo $port_type
35} 35}
36 36
37function get_num_probe_vfs () { 37function get_num_probe_vfs () {
38 if [ $DRIVER == 'mlx4_en' ]; then 38 if [ `get_port_type` -eq "2" ]; then
39 probe_vfs=`calculate_total_vfs` 39 probe_vfs=`calculate_total_vfs`
40 else 40 else
41 probe_vfs=0 41 probe_vfs=0
@@ -65,7 +65,7 @@ function calculate_total_vfs () {
65 fi 65 fi
66 66
67 # Set Ethernet RDMA storage network 67 # Set Ethernet RDMA storage network
68 if [ $ISER == true ] && [ $DRIVER == 'mlx4_en' ] \ 68 if [ $ISER == true ] && [ `get_port_type` -eq "2" ] \
69 && [ $num_of_vfs -eq 0 ]; then 69 && [ $num_of_vfs -eq 0 ]; then
70 num_of_vfs=1 70 num_of_vfs=1
71 fi 71 fi
@@ -149,37 +149,42 @@ function set_kernel_params () {
149 149
150function burn_vfs_in_fw () { 150function burn_vfs_in_fw () {
151 total_vfs=$1 151 total_vfs=$1
152 # required for mlxconfig to discover mlnx devices 152 if [ $CX == 'ConnectX-3' ]; then
153 service openibd start &>/dev/null 153 # required for mlxconfig to discover mlnx devices
154 service mst start &>/dev/null 154 service openibd start &>/dev/null
155 devices=$(mst status -v | grep $CX| grep pciconf | awk '{print $2}') 155 service mst start &>/dev/null
156 for dev in $devices; do 156 devices=$(mst status -v | grep $CX| grep pciconf | awk '{print $2}')
157 logger_print debug "device=$dev" 157 for dev in $devices; do
158 mlxconfig -d $dev q | grep SRIOV | awk '{print $2}' | grep $SRIOV_ENABLED_FLAG &>/dev/null 158 logger_print debug "device=$dev"
159 sriov_enabled=$? 159 mlxconfig -d $dev q | grep SRIOV | awk '{print $2}' | grep $SRIOV_ENABLED_FLAG &>/dev/null
160 current_num_of_vfs=`mlxconfig -d $dev q | grep NUM_OF_VFS | awk '{print $2}'` 160 sriov_enabled=$?
161 if [ $sriov_enabled -eq 0 ] 2>/dev/null; then 161 current_num_of_vfs=`mlxconfig -d $dev q | grep NUM_OF_VFS | awk '{print $2}'`
162 logger_print debug "Detected SR-IOV is already enabled" 162 if [ $sriov_enabled -eq 0 ] 2>/dev/null; then
163 else 163 logger_print debug "Detected SR-IOV is already enabled"
164 logger_print debug "Detected SR-IOV is disabled" 164 else
165 fi 165 logger_print debug "Detected SR-IOV is disabled"
166 if [[ ! "$total_vfs" == "$current_num_of_vfs" ]] 2>/dev/null; then
167 logger_print debug "Current allowed number of VFs is ${current_num_of_vfs}, required number is ${total_vfs}"
168 logger_print debug "Trying mlxconfig -y -d ${dev} s SRIOV_EN=1 NUM_OF_VFS=${total_vfs}"
169 mlxconfig -y -d $dev s SRIOV_EN=1 NUM_OF_VFS=$total_vfs 2>&1 >/dev/null
170 if [ $? -ne 0 ]; then
171 logger_print error "Failed changing number of VFs in FW for HCA ${dev}"
172 fi 166 fi
173 else 167 if [[ ! "$total_vfs" == "$current_num_of_vfs" ]] 2>/dev/null; then
174 logger_print debug "Current number of VFs is correctly set to ${current_num_of_vfs} in FW." 168 logger_print debug "Current allowed number of VFs is ${current_num_of_vfs}, required number is ${total_vfs}"
175 fi 169 logger_print debug "Trying mlxconfig -y -d ${dev} s SRIOV_EN=1 NUM_OF_VFS=${total_vfs}"
176 done 170 mlxconfig -y -d $dev s SRIOV_EN=1 NUM_OF_VFS=$total_vfs 2>&1 >/dev/null
177 service mst stop &>/dev/null 171 if [ $? -ne 0 ]; then
172 logger_print error "Failed changing number of VFs in FW for HCA ${dev}"
173 fi
174 else
175 logger_print debug "Current number of VFs is correctly set to ${current_num_of_vfs} in FW."
176 fi
177 done
178 service mst stop &>/dev/null
179 fi
180 if [ $CX == 'ConnectX-4' ]; then
181 logger_print debug "Skipping burning ConnectX-4 as it is burnt in bootstrap stage."
182 fi
178} 183}
179 184
180function is_sriov_required () { 185function is_sriov_required () {
181 [ $SRIOV == true ] || 186 [ $SRIOV == true ] ||
182 ( [ $ISER == true ] && [ $DRIVER == 'mlx4_en' ] ) 187 ( [ $ISER == true ] && [ `get_port_type` -eq "2" ] )
183 return $? 188 return $?
184} 189}
185 190
@@ -195,9 +200,18 @@ function configure_sriov () {
195 200
196 probe_vfs=`get_num_probe_vfs` 201 probe_vfs=`get_num_probe_vfs`
197 port_type=`get_port_type` 202 port_type=`get_port_type`
198 set_modprobe_file $total_vfs &&
199 set_kernel_params && 203 set_kernel_params &&
200 burn_vfs_in_fw $total_vfs 204 burn_vfs_in_fw $total_vfs
205 if [ $CX == 'ConnectX-3' ]; then
206 set_modprobe_file $total_vfs &&
207 logger_print info "Detected: ConnectX-3 card"
208 fi
209
210 if [ $CX == 'ConnectX-4' ]; then
211 set_sriov $total_vfs &&
212 logger_print info "Detected: ConnectX-4 card"
213 fi
214
201 return $? 215 return $?
202 else 216 else
203 logger_print info "Skipping SR-IOV configuration" 217 logger_print info "Skipping SR-IOV configuration"
@@ -236,8 +250,15 @@ function validate_sriov () {
236 logger_print error "Failed , trying to fallback to ${FALLBACK_NUM_VFS}" 250 logger_print error "Failed , trying to fallback to ${FALLBACK_NUM_VFS}"
237 probe_vfs=`get_num_probe_vfs` 251 probe_vfs=`get_num_probe_vfs`
238 port_type=`get_port_type` 252 port_type=`get_port_type`
239 set_modprobe_file $FALLBACK_NUM_VFS 253
240 service openibd restart &> /dev/null 254 if [ $CX == 'ConnectX-3' ]; then
255 set_modprobe_file $FALLBACK_NUM_VFS
256 service openibd restart &> /dev/null
257 fi
258 if [ $CX == 'ConnectX-4' ]; then
259 set_sriov $FALLBACK_NUM_VFS
260 fi
261
241 current_num_vfs=`lspci | grep -i mellanox | grep -i virtual | wc -l` 262 current_num_vfs=`lspci | grep -i mellanox | grep -i virtual | wc -l`
242 if [ $current_num_vfs -eq $FALLBACK_NUM_VFS ]; then 263 if [ $current_num_vfs -eq $FALLBACK_NUM_VFS ]; then
243 logger_print info "Fallback to ${FALLBACK_NUM_VFS} succeeded" 264 logger_print info "Fallback to ${FALLBACK_NUM_VFS} succeeded"
@@ -248,11 +269,38 @@ function validate_sriov () {
248 fi 269 fi
249} 270}
250 271
272function set_sriov () {
273 PORT_TYPE=`get_port_type`
274 TOTAL_VFS=$1
275 device_up=$PHYSICAL_PORT
276
277 if [ ${#device_up} -eq 0 ]; then
278 logger_print error "Failed to find mlx5 up ports in ibdev2netdev."
279 exit 1
280 else
281 res=`echo 0 > /sys/class/net/${device_up}/device/mlx5_num_vfs`
282 res=`echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs`
283 if [ ! $? -eq 0 ]; then
284 logger_print error "Failed to write $TOTAL_VFS > /sys/class/net/${device_up}/device/mlx5_num_vfs"
285 exit 1
286 fi
287 echo "#!/bin/bash" > /etc/network/if-up.d/sriov_vfs
288 echo "echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs" >> /etc/network/if-up.d/sriov_vfs
289 chmod +x /etc/network/if-up.d/sriov_vfs
290 ifup --all
291 if [ ! $? -eq 0 ]; then
292 logger_print error "Failed to write $TOTAL_VFS > /sys/class/net/${device_up}/device/mlx5_num_vfs"
293 exit 1
294 else
295 logger_print debug "Configured total vfs ${TOTAL_VFS} on ${device_up}"
296 fi
297 fi
298}
251################# 299#################
252 300
253case $SCRIPT_MODE in 301case $SCRIPT_MODE in
254 'configure') 302 'configure')
255 configure_sriov 303 configure_sriov
256 ;; 304 ;;
257 'validate') 305 'validate')
258 validate_sriov 306 validate_sriov
diff --git a/deployment_scripts/vxlan_offloading.sh b/deployment_scripts/vxlan_offloading.sh
index 5b5b546..525e651 100755
--- a/deployment_scripts/vxlan_offloading.sh
+++ b/deployment_scripts/vxlan_offloading.sh
@@ -20,16 +20,16 @@ source $SCRIPT_DIR/common
20readonly SCRIPT_MODE=$1 20readonly SCRIPT_MODE=$1
21 21
22function get_port_type() { 22function get_port_type() {
23 if [ $DRIVER == 'mlx4_en' ]; then 23 if [ $NETWORK_TYPE == 'ethernet' ]; then
24 port_type=2 24 port_type=2
25 elif [ $DRIVER == 'eth_ipoib' ]; then 25 else
26 port_type=1 26 port_type=1
27 fi 27 fi
28 echo $port_type 28 echo $port_type
29} 29}
30 30
31function is_vxlan_offloading_required () { 31function is_vxlan_offloading_required () {
32 [ $VXLAN_OFFLOADING == true ] && [ $DRIVER == 'mlx4_en' ] 32 [ $VXLAN_OFFLOADING == true ] && [ $NETWORK_TYPE == 'ethernet' ]
33 return $? 33 return $?
34} 34}
35 35
@@ -69,7 +69,13 @@ function set_modprobe_file () {
69configure_vxlan_offloading 69configure_vxlan_offloading
70case $SCRIPT_MODE in 70case $SCRIPT_MODE in
71 'configure') 71 'configure')
72 configure_vxlan_offloading 72 if [ "$CX" == "ConnectX3" ]; then
73 configure_vxlan_offloading
74 fi
75 if [ "$CX" == "ConnectX4" ]; then
76 logger_print info "Skipping VXLAN configuration because VXLAN is configured with ConnectX4."
77 fi
78
73 ;; 79 ;;
74 'validate') 80 'validate')
75 # to be added later. 81 # to be added later.
diff --git a/deployment_tasks.yaml b/deployment_tasks.yaml
index 1f2b0ca..dd322c0 100644
--- a/deployment_tasks.yaml
+++ b/deployment_tasks.yaml
@@ -63,17 +63,27 @@
63# Install OFED + FW upgrade 63# Install OFED + FW upgrade
64- id: install_mlnx_ofed 64- id: install_mlnx_ofed
65 role: '*' 65 role: '*'
66 required_for: [configure_vxlan_offload] 66 required_for: [customize_mellanox_hiera_post]
67 requires: [validate_kernel_devel] 67 requires: [validate_kernel_devel]
68 type: shell 68 type: shell
69 parameters: 69 parameters:
70 cmd: ./install_ofed.sh 70 cmd: ./install_ofed.sh
71 timeout: 5000 71 timeout: 5000
72# Add relevant settings for Mellanox manifests to mellanox plugin section in
73# Hiera, to make the data easily accessible and independent of astute.yaml
74- id: customize_mellanox_hiera_post
75 role: '*'
76 required_for: [configure_vxlan_offload]
77 requires: [install_mlnx_ofed]
78 type: shell
79 parameters:
80 cmd: ./mellanox_settings.py
81 timeout: 100
72# add VXLAN offloading parameters 82# add VXLAN offloading parameters
73- id: configure_vxlan_offload 83- id: configure_vxlan_offload
74 role: '*' 84 role: '*'
75 required_for: [configure_sriov] 85 required_for: [configure_sriov]
76 requires: [install_mlnx_ofed] 86 requires: [customize_mellanox_hiera_post]
77 type: shell 87 type: shell
78 parameters: 88 parameters:
79 cmd: ./vxlan_offloading.sh configure 89 cmd: ./vxlan_offloading.sh configure
@@ -87,34 +97,26 @@
87 type: shell 97 type: shell
88 parameters: 98 parameters:
89 cmd: ./sriov.sh configure 99 cmd: ./sriov.sh configure
90 timeout: 200 100 timeout: 300
91# if VXLAN is not required. Rename iSER interface for Eth mode 101# if VXLAN is not required. Rename iSER interface for Eth mode
92- id: rename_iser_probe_vf 102- id: rename_iser_probe_vf
93 role: '*' 103 role: '*'
94 required_for: [reboot_after_ofed] 104 required_for: [validate_sriov]
95 requires: [configure_sriov] 105 requires: [configure_sriov]
96 type: puppet 106 type: puppet
97 parameters: 107 parameters:
98 puppet_manifest: puppet/manifests/iser_rename.pp 108 puppet_manifest: puppet/manifests/iser_rename.pp
99 puppet_modules: puppet/modules:/etc/puppet/modules 109 puppet_modules: puppet/modules:/etc/puppet/modules
100 timeout: 200 110 timeout: 300
101# Reboot due to OFED installation / IOMMU configuration
102- id: reboot_after_ofed
103 role: '*'
104 required_for: [validate_sriov]
105 requires: [rename_iser_probe_vf]
106 type: reboot
107 parameters:
108 timeout: 5000
109# Check number of VFs 111# Check number of VFs
110- id: validate_sriov 112- id: validate_sriov
111 role: '*' 113 role: '*'
112 required_for: [configure_qos] 114 required_for: [configure_qos]
113 requires: [reboot_after_ofed] 115 requires: [rename_iser_probe_vf]
114 type: shell 116 type: shell
115 parameters: 117 parameters:
116 cmd: ./sriov.sh validate 118 cmd: ./sriov.sh validate
117 timeout: 200 119 timeout: 300
118# Enable QoS if required 120# Enable QoS if required
119- id: configure_qos 121- id: configure_qos
120 role: '*' 122 role: '*'
@@ -162,7 +164,7 @@
162 type: shell 164 type: shell
163 parameters: 165 parameters:
164 cmd: ./post_set_mellanox_interfaces.sh 166 cmd: ./post_set_mellanox_interfaces.sh
165 timeout: 200 167 timeout: 500
166# Execute post_deployment manifest for each role 168# Execute post_deployment manifest for each role
167- id: configure_mellanox_controller 169- id: configure_mellanox_controller
168 role: ['controller', 'primary-controller'] 170 role: ['controller', 'primary-controller']
diff --git a/environment_config.yaml b/environment_config.yaml
index 218a2f3..e9646d4 100644
--- a/environment_config.yaml
+++ b/environment_config.yaml
@@ -32,7 +32,7 @@ attributes:
32 If selected, Neutron "Quality of Service" (QoS) will be enabled for SR-IOV ports over Mellanox HCAs. 32 If selected, Neutron "Quality of Service" (QoS) will be enabled for SR-IOV ports over Mellanox HCAs.
33 This feature is supported in Ethernet mode over Neutron with VLAN after SR-IOV direct port creation. 33 This feature is supported in Ethernet mode over Neutron with VLAN after SR-IOV direct port creation.
34 weight: 30 34 weight: 30
35 type: "checkbox" 35 type: hidden
36 restrictions: 36 restrictions:
37 - condition: "settings:common.libvirt_type.value != 'kvm' or not (settings:mellanox-plugin.sriov.value == true) or not (cluster:net_provider == 'neutron' and networking_parameters:segmentation_type == 'vlan')" 37 - condition: "settings:common.libvirt_type.value != 'kvm' or not (settings:mellanox-plugin.sriov.value == true) or not (cluster:net_provider == 'neutron' and networking_parameters:segmentation_type == 'vlan')"
38 message: "For Mellanox QoS support in Neutron, hypervisor type should be KVM and the environment networking should be based on Neutron with VLAN segmentation with Mellanox SR-IOV support." 38 message: "For Mellanox QoS support in Neutron, hypervisor type should be KVM and the environment networking should be based on Neutron with VLAN segmentation with Mellanox SR-IOV support."
diff --git a/metadata.yaml b/metadata.yaml
index 915750b..ee3534a 100644
--- a/metadata.yaml
+++ b/metadata.yaml
@@ -2,16 +2,16 @@
2name: mellanox-plugin 2name: mellanox-plugin
3 3
4# Human-readable name for your plugin 4# Human-readable name for your plugin
5title: Mellanox Openstack Features 5title: Mellanox ConnectX-4 Openstack Features
6 6
7# Plugin version 7# Plugin version
8version: 3.0.0 8version: 3.1.0
9 9
10# Description 10# Description
11description: Enable features over Mellanox hardware 11description: Enable features over Mellanox ConnectX-4 Adapters
12 12
13# Required fuel version 13# Required fuel version
14fuel_version: ['7.0', '8.0'] 14fuel_version: ['8.0']
15 15
16# Specify license of your plugin 16# Specify license of your plugin
17licenses: ['Apache License Version 2.0'] 17licenses: ['Apache License Version 2.0']
@@ -28,11 +28,6 @@ groups: ['storage::cinder', 'network', 'hypervisor']
28# The plugin is compatible with releases in the list 28# The plugin is compatible with releases in the list
29releases: 29releases:
30 - os: ubuntu 30 - os: ubuntu
31 version: 2015.1.0-7.0
32 mode: ['ha']
33 deployment_scripts_path: deployment_scripts/
34 repository_path: repositories/ubuntu
35 - os: ubuntu
36 version: liberty-8.0 31 version: liberty-8.0
37 mode: ['ha'] 32 mode: ['ha']
38 deployment_scripts_path: deployment_scripts/ 33 deployment_scripts_path: deployment_scripts/
diff --git a/pre_build_hook b/pre_build_hook
index e7b851a..d22f900 100755
--- a/pre_build_hook
+++ b/pre_build_hook
@@ -72,7 +72,7 @@ old_debs="${PLUGIN_DIR}/repositories/ubuntu/*.deb"
72deb_files="cirros-testvm-mellanox_0.3.2-ubuntu3_amd64.deb 72deb_files="cirros-testvm-mellanox_0.3.2-ubuntu3_amd64.deb
73 cirros-testvm-mellanox-ib_0.3.2-9_amd64.deb 73 cirros-testvm-mellanox-ib_0.3.2-9_amd64.deb
74 eswitchd_1.0.0-18_amd64.deb 74 eswitchd_1.0.0-18_amd64.deb
75 mlnx-ofed-fuel_3.1-1.5.5_amd64.deb 75 mlnx-ofed-fuel_3.2-2.0.0.0_amd64.deb
76 mlnx-ofed-kernel-dkms_3.1-OFED.3.1.1.0.3.1.g9032737_all.deb 76 mlnx-ofed-kernel-dkms_3.1-OFED.3.1.1.0.3.1.g9032737_all.deb
77 mlnx-ofed-kernel-utils_3.1-OFED.3.1.1.0.3.1.g9032737_amd64.deb 77 mlnx-ofed-kernel-utils_3.1-OFED.3.1.1.0.3.1.g9032737_amd64.deb
78 lldpd_0.9.1-0_amd64.deb 78 lldpd_0.9.1-0_amd64.deb