summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJenkins <jenkins@review.openstack.org>2016-05-29 11:45:13 +0000
committerGerrit Code Review <review@openstack.org>2016-05-29 11:45:13 +0000
commit1c6f8997ba55942dec76415e648e57c328022892 (patch)
tree77185293b5f6b8ced86751f22455b0a69756b79c
parent336e3abb87f44bd4ce31764b60423312383179a6 (diff)
parent8b30f4cb751c624f4d6a3db1ad3f11a83caed4c6 (diff)
Merge "Set VFs MAC addresses"
-rwxr-xr-xdeployment_scripts/configure_mellanox_vfs.py204
-rwxr-xr-xdeployment_scripts/install_ofed.sh8
-rw-r--r--deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb3
-rwxr-xr-xdeployment_scripts/sriov.sh44
-rwxr-xr-xpre_build_hook2
5 files changed, 241 insertions, 20 deletions
diff --git a/deployment_scripts/configure_mellanox_vfs.py b/deployment_scripts/configure_mellanox_vfs.py
new file mode 100755
index 0000000..b6d31b1
--- /dev/null
+++ b/deployment_scripts/configure_mellanox_vfs.py
@@ -0,0 +1,204 @@
1#!/usr/bin/env python
2# Copyright 2016 Mellanox Technologies, Ltd
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13# implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17import os
18import sys
19import subprocess
20import logging
21import traceback
22import glob
23
24LOG_FILE = '/var/log/mellanox-plugin.log'
25
26class MellanoxVfsSettingsException(Exception):
27 pass
28
29class MellanoxVfsSettings(object):
30
31 mellanox_vfs = list();
32
33 @classmethod
34 def get_pci_list(cls):
35 cmd_vfs = "lspci -D | grep -i mellanox | grep -i virtual | awk '{print $1}'"
36 p_lspci = subprocess.Popen(cmd_vfs, shell=True, stdout=subprocess.PIPE,
37 stderr=subprocess.STDOUT)
38 retval = p_lspci.wait()
39 if retval != 0:
40 print 'Failed to execute command lspci.'
41 logging.error("Failed to find mellanox VFs.")
42 sys.exit(1)
43 pci_res = list()
44 for line in p_lspci.stdout.readlines():
45 pci_res.append(line.rstrip())
46 return pci_res
47
48 @classmethod
49 def build_vfs_dict(cls):
50 pci_res = cls.get_pci_list()
51 cmd_vfs_data = "ibdev2netdev -v"
52 p_vfs_data = subprocess.Popen(cmd_vfs_data, shell=True, stdout=subprocess.PIPE,
53 stderr=subprocess.STDOUT)
54 retval = p_vfs_data.wait()
55 if retval != 0:
56 print 'Failed to execute command ibdev2netdev -v.'
57 logging.error("Failed to execute ibdev2netdev -v")
58 sys.exit(1)
59
60 ibdev = list()
61 for vf_data in p_vfs_data.stdout.readlines():
62 ibdev.append(vf_data.rstrip())
63
64 cmd_vfs_macs = "ip link show | grep vf | awk '{print $4}' | tr -d ','"
65 p_vfs_macs = subprocess.Popen(cmd_vfs_macs, shell=True, stdout=subprocess.PIPE,
66 stderr=subprocess.STDOUT)
67 retval = p_vfs_macs.wait()
68 if retval != 0:
69 print 'Failed to execute command ip link show | grep vf '
70 logging.error("Failed to find VFs in ip link show command")
71 sys.exit(1)
72
73 mac_list = list()
74 for mac in p_vfs_macs.stdout.readlines():
75 mac_list.append(mac.rstrip())
76
77 count = 0
78 for pci_address in pci_res:
79 vf_dict = dict();
80 vf_info = next((vf_info for vf_info in ibdev if pci_address in vf_info), None)
81 vf_info = vf_info.split();
82 vf_dict['vf_num'] = count;
83 vf_dict['pci_address'] = vf_info[0]
84 vf_dict['port_module'] = vf_info[1]
85 vf_dict['port_num'] = vf_info[12]
86 vf_dict['mac'] = mac_list[count]
87 cls.mellanox_vfs.append(vf_dict)
88 count += 1;
89
90 @classmethod
91 def unbind(cls):
92 for vf in cls.mellanox_vfs:
93 cmd_vfs_pci = "echo {0} ".format(vf['pci_address']) + \
94 ">> /sys/bus/pci/drivers/mlx5_core/unbind"
95 p_unbind_pci = subprocess.Popen(cmd_vfs_pci, shell=True, stdout=subprocess.PIPE,
96 stderr=subprocess.STDOUT)
97 retval = p_unbind_pci.wait()
98 if retval != 0:
99 logging.error("Failed to unbind pci address {0}".format(vf['pci_address']))
100 sys.exit(1)
101 logging.info("Managed to unbind VFs.")
102
103 @classmethod
104 def bind(cls):
105 for vf in cls.mellanox_vfs:
106 cmd_vfs_pci = "echo {0} ".format(vf['pci_address']) + \
107 ">> /sys/bus/pci/drivers/mlx5_core/bind"
108 p_bind_pci = subprocess.Popen(cmd_vfs_pci, shell=True, stdout=subprocess.PIPE,
109 stderr=subprocess.STDOUT)
110 retval = p_bind_pci.wait()
111 if retval != 0:
112 print 'Bind: Error is:', p_bind_pci.stdout.readlines()
113 logging.error("Failed to bind pci address {0}".format(vf['pci_address']) )
114 sys.exit(1)
115 else:
116 print 'Managed to bind pci address ', vf['pci_address']
117 logging.debug("Managed to bind pci address {0}".format(vf['pci_address']) )
118 logging.info("Managed to bind VFs.")
119
120 @classmethod
121 def assign_mac_per_vf(cls):
122 for vf in cls.mellanox_vfs:
123 if "00:00:00:00:00:00" in vf['mac']:
124 cmd_generate_mac = "ibstat {0} {1} |".format(vf['port_module'], vf['port_num']) + \
125 " grep GUID | cut -d' ' -f3 | cut -d'x' -f2"
126 p_cmd_generate_mac = subprocess.Popen(cmd_generate_mac, shell=True,
127 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
128 retval = p_cmd_generate_mac.wait()
129 if retval != 0:
130 print 'Failed to get ibstat port guid'
131 logging.error("Failed to find PORT GUID to set it as a MAC address for the VF")
132 sys.exit(1)
133 port_guid = p_cmd_generate_mac.stdout.readlines()[0].rstrip();
134 port_guid_to_mac = port_guid[0:12]
135 port_guid_to_mac = ':'.join(port_guid_to_mac[i:i+2] for i in range \
136 (0, len(port_guid_to_mac), 2))
137 vf['mac'] = port_guid_to_mac
138
139 @classmethod
140 def set_mac_per_vf(cls):
141 cmd_physical_port = "hiera mellanox-plugin | grep physical_port | cut -d'>' -f2 "
142 p = subprocess.Popen(cmd_physical_port ,shell=True,stdout=subprocess.PIPE,
143 stderr=subprocess.STDOUT)
144 physical_port = p.stdout.readlines()[0].rstrip()
145 physical_port = physical_port.strip(',"')
146 for vf in cls.mellanox_vfs:
147 cmd_set_mac_per_vf = "ip link set " + \
148 "{0} vf {1} mac {2}".format(physical_port,vf['vf_num'], vf['mac'])
149 p_cmd_set_mac_per_vf = subprocess.Popen(cmd_set_mac_per_vf,shell=True,
150 stdout=subprocess.PIPE,
151 stderr=subprocess.STDOUT)
152 retval = p_cmd_set_mac_per_vf.wait()
153 if retval != 0:
154 print 'Failed to set vf mac', cmd_set_mac_per_vf
155 logging.error("Failed to set MAC address to VF {0}".format(vf['vf_num']))
156 sys.exit(1)
157
158 @classmethod
159 def wait_for_mlx_modules_and_vfs_loaded(cls, total_vfs):
160 retval = 1
161 while (retval != 0):
162 cmd_lsmod = "lsmod | grep mlx_compat"
163 p_lsmod = subprocess.Popen(cmd_lsmod, shell=True,stdout=subprocess.PIPE,
164 stderr=subprocess.STDOUT)
165 retval = p_lsmod.wait()
166 logging.info("mlx_compat module is loaded!")
167
168 number_of_vfs = 0
169 while ( number_of_vfs != total_vfs):
170 cmd_number_vfs = "lspci | grep -i mellanox | grep -i virtual | wc -l"
171 p_number_vfs = subprocess.Popen(cmd_number_vfs, shell=True,stdout=subprocess.PIPE,
172 stderr=subprocess.STDOUT)
173 retval = p_number_vfs.wait()
174 if retval != 0:
175 logging.error("Failed to run lspci")
176 sys.exit(1)
177 number_of_vfs = int(p_number_vfs.stdout.readlines()[0].rstrip());
178 logging.info("all VFs are loaded.")
179
180def main(total_vfs):
181 logging.basicConfig(format='%(asctime)s %(message)s',
182 level=logging.DEBUG, filename=LOG_FILE)
183 try:
184 vfs_configurations = MellanoxVfsSettings()
185 vfs_configurations.wait_for_mlx_modules_and_vfs_loaded(int(total_vfs))
186 vfs_configurations.build_vfs_dict()
187 vfs_configurations.assign_mac_per_vf()
188 vfs_configurations.unbind()
189 vfs_configurations.set_mac_per_vf()
190 vfs_configurations.bind()
191
192 except MellanoxVfsSettingsException, exc:
193 error_msg = "Failed configuring Mellanox vfs: {0}\n".format(exc)
194 sys.stderr.write(error_msg)
195 logging.error(exc)
196 sys.exit(1)
197 success_msg = "Done configuring Mellanox vfs\n"
198 sys.stdout.write(success_msg)
199 logging.info(success_msg)
200 sys.exit(0)
201
202
203if __name__=="__main__":
204 main(sys.argv[1])
diff --git a/deployment_scripts/install_ofed.sh b/deployment_scripts/install_ofed.sh
index 911fc19..f7dcf44 100755
--- a/deployment_scripts/install_ofed.sh
+++ b/deployment_scripts/install_ofed.sh
@@ -117,6 +117,10 @@ function enable_eipoib (){
117 \cp -f ./openibd /etc/init.d/openibd 117 \cp -f ./openibd /etc/init.d/openibd
118} 118}
119 119
120function install_gawk (){
121 apt-get -y install gawk
122}
123
120if ! is_ofed_installed; then 124if ! is_ofed_installed; then
121 # Install mlnx-ofed-fuel rpm/deb package which extracts OFED installation dir 125 # Install mlnx-ofed-fuel rpm/deb package which extracts OFED installation dir
122 install_mlnx_ofed_src 126 install_mlnx_ofed_src
@@ -130,6 +134,10 @@ if ! is_ofed_installed; then
130 134
131 # Enable Ethernet IP Over Infiniband in case of eth_ipoib driver 135 # Enable Ethernet IP Over Infiniband in case of eth_ipoib driver
132 enable_eipoib 136 enable_eipoib
137
138 # Install gawk package for running ibdev2netdev -v
139 install_gawk
140
133fi 141fi
134 142
135# Decrease loglevels for prevent flooding kernel messages to console 143# Decrease loglevels for prevent flooding kernel messages to console
diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb b/deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb
index c957ea2..d4bc2db 100644
--- a/deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb
+++ b/deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb
@@ -99,9 +99,6 @@ if [ $PROBED_PORT_NAME != $ISER_NAME ]; then
99 udevadm trigger 99 udevadm trigger
100 modprobe mlx4_en 100 modprobe mlx4_en
101 modprobe mlx5_ib && modprobe mlx5_core 101 modprobe mlx5_ib && modprobe mlx5_core
102 echo "#!/bin/bash" > /etc/network/if-up.d/iser_ifc
103 echo "ifconfig $ISER_NAME up " >> /etc/network/if-up.d/iser_ifc
104 ifup --all
105 102
106 if [ $? -ne 0 ]; then 103 if [ $? -ne 0 ]; then
107 logger_print error "Mellanox drivers restart failed." 104 logger_print error "Mellanox drivers restart failed."
diff --git a/deployment_scripts/sriov.sh b/deployment_scripts/sriov.sh
index d5787f5..a79c64b 100755
--- a/deployment_scripts/sriov.sh
+++ b/deployment_scripts/sriov.sh
@@ -278,21 +278,33 @@ function set_sriov () {
278 logger_print error "Failed to find mlx5 up ports in ibdev2netdev." 278 logger_print error "Failed to find mlx5 up ports in ibdev2netdev."
279 exit 1 279 exit 1
280 else 280 else
281 res=`echo 0 > /sys/class/net/${device_up}/device/mlx5_num_vfs` 281 if [ "$(lspci | grep -i mellanox | grep -i virtual | wc -l)" -ne "$TOTAL_VFS" ]; then
282 res=`echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs` 282 res=`echo 0 > /sys/class/net/${device_up}/device/mlx5_num_vfs`
283 if [ ! $? -eq 0 ]; then 283 res=`echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs`
284 logger_print error "Failed to write $TOTAL_VFS > /sys/class/net/${device_up}/device/mlx5_num_vfs" 284 if [ ! $? -eq 0 ]; then
285 exit 1 285 logger_print error "Failed to write $TOTAL_VFS > /sys/class/net/${device_up}/device/mlx5_num_vfs"
286 fi 286 exit 1
287 echo "#!/bin/bash" > /etc/network/if-up.d/sriov_vfs 287 fi
288 echo "echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs" >> /etc/network/if-up.d/sriov_vfs 288
289 chmod +x /etc/network/if-up.d/sriov_vfs 289 # Give MACs to created VFs
290 ifup --all 290 python ./configure_mellanox_vfs.py ${TOTAL_VFS}
291 if [ ! $? -eq 0 ]; then 291
292 logger_print error "Failed to write $TOTAL_VFS > /sys/class/net/${device_up}/device/mlx5_num_vfs" 292 # Make number of VFs and their MACs persistent
293 exit 1 293 persistent_ifup_script=/etc/network/if-up.d/persistent_mlnx_params
294 else 294 echo "#!/bin/bash" > $persistent_ifup_script
295 logger_print debug "Configured total vfs ${TOTAL_VFS} on ${device_up}" 295 chmod +x $persistent_ifup_script
296 echo "if ! lspci | grep -i mellanox | grep -i virtual; then" >> $persistent_ifup_script
297 echo "echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs" >> $persistent_ifup_script
298 echo "python /etc/fuel/plugins/mellanox-plugin-*/configure_mellanox_vfs.py ${TOTAL_VFS}" >> $persistent_ifup_script
299 echo "fi" >> $persistent_ifup_script
300 echo "if [ -f /etc/init.d/tgt ]; then /etc/init.d/tgt force-reload; else exit 0; fi" >> $persistent_ifup_script
301
302 if [ ! $? -eq 0 ]; then
303 logger_print error "Failed to write $TOTAL_VFS > /sys/class/net/${device_up}/device/mlx5_num_vfs"
304 exit 1
305 else
306 logger_print debug "Configured total vfs ${TOTAL_VFS} on ${device_up}"
307 fi
296 fi 308 fi
297 fi 309 fi
298} 310}
@@ -300,7 +312,7 @@ function set_sriov () {
300 312
301case $SCRIPT_MODE in 313case $SCRIPT_MODE in
302 'configure') 314 'configure')
303 configure_sriov 315 configure_sriov
304 ;; 316 ;;
305 'validate') 317 'validate')
306 validate_sriov 318 validate_sriov
diff --git a/pre_build_hook b/pre_build_hook
index c08b515..848b306 100755
--- a/pre_build_hook
+++ b/pre_build_hook
@@ -62,7 +62,7 @@ old_debs="${PLUGIN_DIR}/repositories/ubuntu/*.deb"
62deb_files="cirros-testvm-mellanox_0.3.2-ubuntu3_amd64.deb 62deb_files="cirros-testvm-mellanox_0.3.2-ubuntu3_amd64.deb
63 cirros-testvm-mellanox-ib_0.3.2-9_amd64.deb 63 cirros-testvm-mellanox-ib_0.3.2-9_amd64.deb
64 eswitchd_1.0.0-18_amd64.deb 64 eswitchd_1.0.0-18_amd64.deb
65 mlnx-ofed-fuel_3.2-2.0.0.0_amd64.deb 65 mlnx-ofed-fuel_3.3-0.1.0.0_amd64.deb
66 lldpd_0.9.1-0_amd64.deb 66 lldpd_0.9.1-0_amd64.deb
67 python-networking-mlnx_7.0.0-1_all.deb" 67 python-networking-mlnx_7.0.0-1_all.deb"
68get_packages "deb" "$old_debs" "$deb_files" 68get_packages "deb" "$old_debs" "$deb_files"