openstack
/
deb-networking-mlnx
Code Issues Proposed changes

Add support for multi port on the same phynet 

Change-Id: I70db5d405d57430efa3d99034d75133287d7aa4a
This commit is contained in:
Moshe Levi 2017-04-30 21:18:30 +03:00
parent 1175563298
commit 71cfd3a094
8 changed files with 224 additions and 258 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
networking_mlnx/eswitchd/common/constants.py
View File

@ -25,33 +25,31 @@ UNTAGGED_VLAN_ID = 4095
INVALID_MAC = '00:00:00:00:00:00' INVALID_MAC = '00:00:00:00:00:00'
# CX3 # MLNX4
ADMIN_GUID_PATH = "/sys/class/infiniband/%s/iov/ports/%s/admin_guids/%s" MLNX4_ADMIN_GUID_PATH = "/sys/class/infiniband/%s/iov/ports/%s/admin_guids/%s"
GUID_INDEX_PATH = "/sys/class/infiniband/%s/iov/%s/ports/%s/gid_idx/0" MLNX4_GUID_INDEX_PATH = "/sys/class/infiniband/%s/iov/%s/ports/%s/gid_idx/0"
PKEY_INDEX_PATH = "/sys/class/infiniband/%s/iov/%s/ports/%s/pkey_idx/%s" MLNX4_PKEY_INDEX_PATH = "/sys/class/infiniband/%s/iov/%s/ports/%s/pkey_idx/%s"
# CX4 # MLNX5
CX4_GUID_NODE_PATH = ('/sys/class/infiniband/%(module)s/device/sriov/' MLNX5_GUID_NODE_PATH = ('/sys/class/infiniband/%(module)s/device/sriov/'
'%(vf_num)s/node') '%(vf_num)s/node')
CX4_GUID_PORT_PATH = ('/sys/class/infiniband/%(module)s/device/sriov/' MLNX5_GUID_PORT_PATH = ('/sys/class/infiniband/%(module)s/device/sriov/'
'%(vf_num)s/port') '%(vf_num)s/port')
CX4_GUID_POLICY_PATH = ('/sys/class/infiniband/%(module)s/device/sriov/' MLNX5_GUID_POLICY_PATH = ('/sys/class/infiniband/%(module)s/device/sriov/'
'%(vf_num)s/policy') '%(vf_num)s/policy')
UNBIND_PATH = '/sys/bus/pci/drivers/mlx5_core/unbind' UNBIND_PATH = '/sys/bus/pci/drivers/mlx5_core/unbind'
BIND_PATH = '/sys/bus/pci/drivers/mlx5_core/bind' BIND_PATH = '/sys/bus/pci/drivers/mlx5_core/bind'
INVALID_GUID_CX3 = 'ffffffffffffffff' MLNX4_INVALID_GUID = 'ffffffffffffffff'
INVALID_GUID_CX4 = 'ff:ff:ff:ff:ff:ff:ff:ff' MLNX5_INVALID_GUID = 'ff:ff:ff:ff:ff:ff:ff:ff'
CONN_URL = '%(transport)s://%(addr)s:%(port)s' CONN_URL = '%(transport)s://%(addr)s:%(port)s'
CX3_VF_DEVICE_TYPE_LIST = ('0x1004', ) MLNX4_VF_DEVICE_TYPE_LIST = ('0x1004', )
CX4_VF_DEVICE_TYPE_LIST = ('0x1014', '0x1016') MLNX5_VF_DEVICE_TYPE_LIST = ('0x1014', '0x1016', '0x1018')
CX5_VF_DEVICE_TYPE_LIST = ('0x1018', )
CX3_VF_DEVICE_TYPE = 'CX3' MLNX4_VF_DEVICE_TYPE = 'MLNX4'
CX4_VF_DEVICE_TYPE = 'CX4' MLNX5_VF_DEVICE_TYPE = 'MLNX5'
CX5_VF_DEVICE_TYPE = 'CX5'
SOCKET_OS_PORT = '60001' SOCKET_OS_PORT = '60001'
SOCKET_OS_TRANSPORT = 'tcp' SOCKET_OS_TRANSPORT = 'tcp'

38
networking_mlnx/eswitchd/db/device_db.py
View File

@ -26,25 +26,31 @@ class DeviceDB(object):
def add_fabric(self, fabric, pf, pci_id, hca_port, fabric_type, def add_fabric(self, fabric, pf, pci_id, hca_port, fabric_type,
pf_mlx_dev): pf_mlx_dev):
details = {} pf_details = {}
details['vfs'] = {} pf_details['vfs'] = {}
details['pf'] = pf pf_details['pf_device_type'] = None
details['pf_device_type'] = None pf_details['pci_id'] = pci_id
details['pci_id'] = pci_id pf_details['hca_port'] = hca_port
details['hca_port'] = hca_port pf_details['fabric_type'] = fabric_type
details['fabric_type'] = fabric_type pf_details['pf_mlx_dev'] = pf_mlx_dev
details['pf_mlx_dev'] = pf_mlx_dev if self.device_db.get(fabric) is None:
self.device_db[fabric] = details self.device_db[fabric] = {pf: pf_details}
else:
self.device_db[fabric][pf] = pf_details
def get_fabric_details(self, fabric): def get_fabric_details(self, fabric, pf=None):
return self.device_db[fabric] if pf is None:
return self.device_db[fabric]
else:
return self.device_db[fabric][pf]
def set_fabric_devices(self, fabric, vfs): def set_fabric_devices(self, fabric, pf, vfs):
self.device_db[fabric]['vfs'] = vfs self.device_db[fabric][pf]['vfs'] = vfs
vf = six.next(six.itervalues(vfs)) vf = six.next(six.itervalues(vfs))
self.device_db[fabric]['pf_device_type'] = vf['vf_device_type'] self.device_db[fabric][pf]['pf_device_type'] = vf['vf_device_type']
def get_dev_fabric(self, dev): def get_dev_fabric(self, dev):
for fabric in self.device_db: for fabric in self.device_db:
if dev in self.device_db[fabric]['vfs']: for pf in self.device_db[fabric]:
return fabric if dev in self.device_db[fabric][pf]['vfs']:
return fabric

4
networking_mlnx/eswitchd/db/eswitch_db.py
View File

@ -23,9 +23,11 @@ LOG = logging.getLogger(__name__)
class eSwitchDB(object): class eSwitchDB(object):
def __init__(self): def __init__(self, pf, vfs):
self.port_table = {} self.port_table = {}
self.port_policy = {} self.port_policy = {}
self.vfs = vfs
self.pf = pf
def create_port(self, port_name, port_type): def create_port(self, port_name, port_type):
self.port_table.update({port_name: {'type': port_type, self.port_table.update({port_name: {'type': port_type,

236
networking_mlnx/eswitchd/eswitch_handler.py
View File

@ -67,7 +67,11 @@ class eSwitchHandler(object):
sys.exit(1) sys.exit(1)
if fabric_type: if fabric_type:
self.eswitches[fabric] = eswitch_db.eSwitchDB() if self.eswitches.get(fabric) is None:
self.eswitches[fabric] = []
vfs = self.pci_utils.get_vfs_info(pf)
self.eswitches[fabric].append(
eswitch_db.eSwitchDB(pf=pf, vfs=vfs))
self._add_fabric(fabric, pf, fabric_type) self._add_fabric(fabric, pf, fabric_type)
res_fabrics.append((fabric, pf, fabric_type)) res_fabrics.append((fabric, pf, fabric_type))
else: else:
@ -88,19 +92,27 @@ class eSwitchHandler(object):
def _add_fabric(self, fabric, pf, fabric_type): def _add_fabric(self, fabric, pf, fabric_type):
self.rm.add_fabric(fabric, pf, fabric_type) self.rm.add_fabric(fabric, pf, fabric_type)
self._config_port_up(pf) self._config_port_up(pf)
fabric_details = self.rm.get_fabric_details(fabric) pf_fabric_details = self.rm.get_fabric_details(fabric, pf)
eswitches = self._get_eswitches_for_fabric(fabric)
for vf in fabric_details['vfs']: eswitch = None
self.eswitches[fabric].create_port(vf, constants.VIF_TYPE_HOSTDEV) for esw in eswitches:
if esw.pf == pf:
eswitch = esw
break
for vf in pf_fabric_details['vfs']:
eswitch.create_port(vf, constants.VIF_TYPE_HOSTDEV)
def _treat_added_devices(self, devices, vm_ids): def _treat_added_devices(self, devices, vm_ids):
for device in devices: for device in devices:
dev, mac, fabric = device dev, mac, fabric = device
if fabric: if fabric:
self.eswitches[fabric].attach_vnic( for eswitch in self.eswitches[fabric]:
port_name=dev, device_id=vm_ids[dev], vnic_mac=mac) if dev in eswitch.vfs:
if self.eswitches[fabric].vnic_exists(mac): eswitch.attach_vnic(
self.eswitches[fabric].plug_nic(port_name=dev) port_name=dev, device_id=vm_ids[dev], vnic_mac=mac)
if eswitch.vnic_exists(mac):
eswitch.plug_nic(port_name=dev)
break
else: else:
LOG.info(_LI("No Fabric defined for device %s"), dev) LOG.info(_LI("No Fabric defined for device %s"), dev)
@ -108,17 +120,20 @@ class eSwitchHandler(object):
for dev, mac in devices: for dev, mac in devices:
fabric = self.rm.get_fabric_for_dev(dev) fabric = self.rm.get_fabric_for_dev(dev)
if fabric: if fabric:
self.eswitches[fabric].detach_vnic(vnic_mac=mac) for eswitch in self.eswitches[fabric]:
if dev in eswitch.vfs:
eswitch.detach_vnic(vnic_mac=mac)
else: else:
LOG.info(_LI("No Fabric defined for device %s"), dev) LOG.info(_LI("No Fabric defined for device %s"), dev)
def get_vnics(self, fabrics): def get_vnics(self, fabrics):
vnics = {} vnics = {}
for fabric in fabrics: for fabric in fabrics:
eswitch = self._get_vswitch_for_fabric(fabric) eswitches = self._get_eswitches_for_fabric(fabric)
if eswitch: if eswitches:
vnics_for_eswitch = eswitch.get_attached_vnics() for eswitch in eswitches:
vnics.update(vnics_for_eswitch) vnics_for_eswitch = eswitch.get_attached_vnics()
vnics.update(vnics_for_eswitch)
else: else:
LOG.error(_LE("No eSwitch found for Fabric %s"), fabric) LOG.error(_LE("No eSwitch found for Fabric %s"), fabric)
continue continue
@ -126,7 +141,7 @@ class eSwitchHandler(object):
return vnics return vnics
def create_port(self, fabric, vnic_type, device_id, vnic_mac, pci_slot): def create_port(self, fabric, vnic_type, device_id, vnic_mac, pci_slot):
eswitch = self._get_vswitch_for_fabric(fabric) eswitch = self._get_eswitch_for_fabric_and_pci(fabric, pci_slot)
if eswitch: if eswitch:
try: try:
if eswitch.attach_vnic( if eswitch.attach_vnic(
@ -143,7 +158,7 @@ class eSwitchHandler(object):
return pci_slot return pci_slot
def plug_nic(self, fabric, device_id, vnic_mac, pci_slot): def plug_nic(self, fabric, device_id, vnic_mac, pci_slot):
eswitch = self._get_vswitch_for_fabric(fabric) eswitch = self._get_eswitch_for_fabric_and_pci(fabric, pci_slot)
if eswitch: if eswitch:
eswitch.port_table[pci_slot]['vnic'] = vnic_mac eswitch.port_table[pci_slot]['vnic'] = vnic_mac
eswitch.port_policy.update( eswitch.port_policy.update(
@ -161,110 +176,129 @@ class eSwitchHandler(object):
def delete_port(self, fabric, vnic_mac): def delete_port(self, fabric, vnic_mac):
dev = None dev = None
eswitch = self._get_vswitch_for_fabric(fabric) eswitches = self._get_eswitches_for_fabric(fabric)
if eswitch: for eswitch in eswitches:
dev = eswitch.detach_vnic(vnic_mac) if eswitch and vnic_mac in eswitch.get_attached_vnics():
if dev: dev = eswitch.detach_vnic(vnic_mac)
self._config_vf_mac_address(fabric, dev) if dev:
else: self._config_vf_mac_address(fabric, dev)
break
if dev is None:
LOG.warning(_LW("No eSwitch found for Fabric %s"), fabric) LOG.warning(_LW("No eSwitch found for Fabric %s"), fabric)
return dev return dev
def port_release(self, fabric, vnic_mac): def port_release(self, fabric, vnic_mac):
ret = None ret = None
eswitch = self._get_vswitch_for_fabric(fabric) dev = None
dev = eswitch.get_dev_for_vnic(vnic_mac) eswitches = self._get_eswitches_for_fabric(fabric)
for eswitch in eswitches:
if eswitch and vnic_mac in eswitch.get_attached_vnics():
dev = eswitch.get_dev_for_vnic(vnic_mac)
break
if dev: if dev:
if (eswitch.get_port_state(dev) == if (eswitch.get_port_state(dev) ==
constants.VPORT_STATE_UNPLUGGED): constants.VPORT_STATE_UNPLUGGED):
ret = self.set_vlan( ret = self.set_vlan(
fabric, vnic_mac, constants.UNTAGGED_VLAN_ID) fabric, vnic_mac, constants.UNTAGGED_VLAN_ID)
self.port_down(fabric, vnic_mac) self.port_down(fabric, vnic_mac)
eswitch = self._get_vswitch_for_fabric(fabric) eswitch.port_release(vnic_mac)
eswitch.port_release(vnic_mac)
return ret return ret
def port_up(self, fabric, vnic_mac): def port_up(self, fabric, vnic_mac):
eswitch = self._get_vswitch_for_fabric(fabric) dev = None
if eswitch: eswitches = self._get_eswitches_for_fabric(fabric)
dev = eswitch.get_dev_for_vnic(vnic_mac) for eswitch in eswitches:
if not dev: if eswitch and vnic_mac in eswitch.get_attached_vnics():
LOG.info(_LI("No device for MAC %s"), vnic_mac) dev = eswitch.get_dev_for_vnic(vnic_mac)
break
if not dev:
LOG.info(_LI("No device for MAC %s"), vnic_mac)
def port_down(self, fabric, vnic_mac): def port_down(self, fabric, vnic_mac):
eswitch = self._get_vswitch_for_fabric(fabric) dev = None
if eswitch: eswitches = self._get_eswitches_for_fabric(fabric)
dev = eswitch.get_dev_for_vnic(vnic_mac) for eswitch in eswitches:
if dev: if eswitch and vnic_mac in eswitch.get_attached_vnics():
LOG.info(_LI("IB port for MAC %s doen't support " dev = eswitch.get_dev_for_vnic(vnic_mac)
"port down"), vnic_mac) if dev:
else: LOG.info(_LI("IB port for MAC %s doen't support "
LOG.info(_LI("No device for MAC %s"), vnic_mac) "port down"), vnic_mac)
break
if dev is None:
LOG.info(_LI("No device for MAC %s"), vnic_mac)
def set_vlan(self, fabric, vnic_mac, vlan): def set_vlan(self, fabric, vnic_mac, vlan):
eswitch = self._get_vswitch_for_fabric(fabric) eswitches = self._get_eswitches_for_fabric(fabric)
if eswitch: for eswitch in eswitches:
eswitch.set_vlan(vnic_mac, vlan) if eswitch and vnic_mac in eswitch.get_attached_vnics():
dev = eswitch.get_dev_for_vnic(vnic_mac) eswitch.set_vlan(vnic_mac, vlan)
state = eswitch.get_port_state(dev) dev = eswitch.get_dev_for_vnic(vnic_mac)
if dev: state = eswitch.get_port_state(dev)
if state in (constants.VPORT_STATE_ATTACHED, if dev:
constants.VPORT_STATE_UNPLUGGED): if state in (constants.VPORT_STATE_ATTACHED,
if eswitch.get_port_table()[dev]['alias']: constants.VPORT_STATE_UNPLUGGED):
dev = eswitch.get_port_table()[dev]['alias'] if eswitch.get_port_table()[dev]['alias']:
try: dev = eswitch.get_port_table()[dev]['alias']
self._config_vlan_ib(fabric, dev, vlan) try:
return True self._config_vlan_ib(fabric, dev, vlan)
except RuntimeError: return True
LOG.error(_LE('Set VLAN operation failed')) except RuntimeError:
LOG.error(_LE('Set VLAN operation failed'))
return False return False
def get_eswitch_tables(self, fabrics): def get_eswitch_tables(self, fabrics):
tables = {} tables = {}
for fabric in fabrics: for fabric in fabrics:
eswitch = self._get_vswitch_for_fabric(fabric) eswitches = self._get_eswitches_for_fabric(fabric)
if eswitch: for eswitch in eswitches:
tables[fabric] = { if eswitch:
'port_table': eswitch.get_port_table_matrix(), tables[fabric] = {
'port_policy': eswitch.get_port_policy_matrix() 'port_table': eswitch.get_port_table_matrix(),
} 'port_policy': eswitch.get_port_policy_matrix()
}
else: else:
LOG.info(_LI("Get eswitch tables: No eswitch %s") % fabric) LOG.info(_LI("Get eswitch tables: No eswitch %s") % fabric)
return tables return tables
def _get_vswitch_for_fabric(self, fabric): def _get_eswitches_for_fabric(self, fabric):
if fabric in self.eswitches: if fabric in self.eswitches:
return self.eswitches[fabric] return self.eswitches[fabric]
else: else:
return return
def _get_eswitch_for_fabric_and_pci(self, fabric, pci_slot):
eswitches = self._get_eswitches_for_fabric(fabric)
for eswitch in eswitches:
if pci_slot in eswitch.vfs:
return eswitch
def _config_vf_pkey(self, ppkey_idx, pkey_idx, def _config_vf_pkey(self, ppkey_idx, pkey_idx,
pf_mlx_dev, vf_pci_id, hca_port): pf_mlx_dev, vf_pci_id, hca_port):
path = constants.PKEY_INDEX_PATH % (pf_mlx_dev, vf_pci_id, path = constants.MLNX4_PKEY_INDEX_PATH % (pf_mlx_dev, vf_pci_id,
hca_port, pkey_idx) hca_port, pkey_idx)
cmd = ['ebrctl', 'write-sys', path, ppkey_idx] cmd = ['ebrctl', 'write-sys', path, ppkey_idx]
command_utils.execute(*cmd) command_utils.execute(*cmd)
def _get_guid_idx(self, pf_mlx_dev, dev, hca_port): def _get_guid_idx(self, pf_mlx_dev, dev, hca_port):
path = constants.GUID_INDEX_PATH % (pf_mlx_dev, dev, hca_port) path = constants.MLNX4_ADMIN_GUID_PATH % (pf_mlx_dev, dev, hca_port)
with open(path) as fd: with open(path) as fd:
idx = fd.readline().strip() idx = fd.readline().strip()
return idx return idx
def _get_guid_from_mac(self, mac, device_type): def _get_guid_from_mac(self, mac, device_type):
guid = None guid = None
if device_type == constants.CX3_VF_DEVICE_TYPE: if device_type == constants.MLNX4_VF_DEVICE_TYPE:
if mac is None: if mac is None:
guid = constants.INVALID_GUID_CX3 guid = constants.MLNX4_INVALID_GUID
else: else:
mac = mac.replace(':', '') mac = mac.replace(':', '')
prefix = mac[:6] prefix = mac[:6]
suffix = mac[6:] suffix = mac[6:]
guid = prefix + '0000' + suffix guid = prefix + '0000' + suffix
elif (device_type == constants.CX4_VF_DEVICE_TYPE or elif (device_type == constants.MLNX5_VF_DEVICE_TYPE):
device_type == constants.CX5_VF_DEVICE_TYPE):
if mac is None: if mac is None:
guid = constants.INVALID_GUID_CX4 guid = constants.MLNX5_INVALID_GUID
else: else:
prefix = mac[:9] prefix = mac[:9]
suffix = mac[9:] suffix = mac[9:]
@ -272,26 +306,26 @@ class eSwitchHandler(object):
return guid return guid
def _config_vf_mac_address(self, fabric, dev, vnic_mac=None): def _config_vf_mac_address(self, fabric, dev, vnic_mac=None):
fabric_details = self.rm.get_fabric_details(fabric) pf_fabric_details = self._get_pf_fabric(fabric, dev)
vf_device_type = fabric_details['vfs'][dev]['vf_device_type'] vf_device_type = pf_fabric_details['vfs'][dev]['vf_device_type']
vguid = self._get_guid_from_mac(vnic_mac, vf_device_type) vguid = self._get_guid_from_mac(vnic_mac, vf_device_type)
if vf_device_type == constants.CX3_VF_DEVICE_TYPE: if vf_device_type == constants.MLNX4_VF_DEVICE_TYPE:
self._config_vf_mac_address_cx3(vguid, dev, fabric_details) self._config_vf_mac_address_mlnx4(vguid, dev, pf_fabric_details)
elif (vf_device_type == constants.CX4_VF_DEVICE_TYPE or elif (vf_device_type == constants.MLNX5_VF_DEVICE_TYPE):
vf_device_type == constants.CX5_VF_DEVICE_TYPE): self._config_vf_mac_address_mlnx5(vguid, dev, pf_fabric_details)
self._config_vf_mac_address_cx4(vguid, dev, fabric_details)
else: else:
LOG.error(_LE("Unsupported vf device type: %s "), LOG.error(_LE("Unsupported vf device type: %s "),
vf_device_type) vf_device_type)
def _config_vf_mac_address_cx3(self, vguid, dev, fabric_details): def _config_vf_mac_address_mlnx4(self, vguid, dev, pf_fabric_details):
hca_port = fabric_details['hca_port'] hca_port = pf_fabric_details['hca_port']
pf_mlx_dev = fabric_details['pf_mlx_dev'] pf_mlx_dev = pf_fabric_details['pf_mlx_dev']
self._config_vf_pkey( self._config_vf_pkey(
INVALID_PKEY, DEFAULT_PKEY_IDX, pf_mlx_dev, dev, hca_port) INVALID_PKEY, DEFAULT_PKEY_IDX, pf_mlx_dev, dev, hca_port)
guid_idx = self._get_guid_idx(pf_mlx_dev, dev, hca_port) guid_idx = self._get_guid_idx(pf_mlx_dev, dev, hca_port)
path = constants.ADMIN_GUID_PATH % (pf_mlx_dev, hca_port, guid_idx) path = constants.MLNX4_ADMIN_GUID_PATH % (
pf_mlx_dev, hca_port, guid_idx)
cmd = ['ebrctl', 'write-sys', path, vguid] cmd = ['ebrctl', 'write-sys', path, vguid]
command_utils.execute(*cmd) command_utils.execute(*cmd)
ppkey_idx = self._get_pkey_idx( ppkey_idx = self._get_pkey_idx(
@ -303,20 +337,20 @@ class eSwitchHandler(object):
LOG.error(_LE("Can't find partial management pkey for" LOG.error(_LE("Can't find partial management pkey for"
"%(pf)s:%(dev)s"), {'pf': pf_mlx_dev, 'dev': dev}) "%(pf)s:%(dev)s"), {'pf': pf_mlx_dev, 'dev': dev})
def _config_vf_mac_address_cx4(self, vguid, dev, fabric_details): def _config_vf_mac_address_mlnx5(self, vguid, dev, pf_fabric_details):
vf_num = fabric_details['vfs'][dev]['vf_num'] vf_num = pf_fabric_details['vfs'][dev]['vf_num']
pf_mlx_dev = fabric_details['pf_mlx_dev'] pf_mlx_dev = pf_fabric_details['pf_mlx_dev']
guid_node = constants.CX4_GUID_NODE_PATH % {'module': pf_mlx_dev, guid_node = constants.MLNX5_GUID_NODE_PATH % {'module': pf_mlx_dev,
'vf_num': vf_num} 'vf_num': vf_num}
guid_port = constants.CX4_GUID_PORT_PATH % {'module': pf_mlx_dev, guid_port = constants.MLNX5_GUID_PORT_PATH % {'module': pf_mlx_dev,
'vf_num': vf_num} 'vf_num': vf_num}
guid_poliy = constants.CX4_GUID_POLICY_PATH % {'module': pf_mlx_dev, guid_poliy = constants.MLNX5_GUID_POLICY_PATH % {'module': pf_mlx_dev,
'vf_num': vf_num} 'vf_num': vf_num}
for path in (guid_node, guid_port): for path in (guid_node, guid_port):
cmd = ['ebrctl', 'write-sys', path, vguid] cmd = ['ebrctl', 'write-sys', path, vguid]
command_utils.execute(*cmd) command_utils.execute(*cmd)
if vguid == constants.INVALID_GUID_CX4: if vguid == constants.MLNX5_INVALID_GUID:
cmd = ['ebrctl', 'write-sys', guid_poliy, 'Down\n'] cmd = ['ebrctl', 'write-sys', guid_poliy, 'Down\n']
command_utils.execute(*cmd) command_utils.execute(*cmd)
cmd = ['ebrctl', 'write-sys', constants.UNBIND_PATH, dev] cmd = ['ebrctl', 'write-sys', constants.UNBIND_PATH, dev]
@ -328,19 +362,19 @@ class eSwitchHandler(object):
command_utils.execute(*cmd) command_utils.execute(*cmd)
def _config_vlan_ib(self, fabric, dev, vlan): def _config_vlan_ib(self, fabric, dev, vlan):
fabric_details = self.rm.get_fabric_details(fabric) pf_fabric_details = self._get_pf_fabric(fabric, dev)
hca_port = fabric_details['hca_port'] hca_port = pf_fabric_details['hca_port']
pf_mlx_dev = fabric_details['pf_mlx_dev'] pf_mlx_dev = pf_fabric_details['pf_mlx_dev']
vf_device_type = fabric_details['vfs'][dev]['vf_device_type'] vf_device_type = pf_fabric_details['vfs'][dev]['vf_device_type']
if vf_device_type == constants.CX3_VF_DEVICE_TYPE: if vf_device_type == constants.MLNX4_VF_DEVICE_TYPE:
self._config_vlan_ib_cx3(vlan, pf_mlx_dev, dev, hca_port) self._config_vlan_ib_mlnx4(vlan, pf_mlx_dev, dev, hca_port)
elif vf_device_type == constants.CX4_VF_DEVICE_TYPE: elif vf_device_type == constants.MLNX5_VF_DEVICE_TYPE:
pass pass
else: else:
LOG.error(_LE("Unsupported vf device type: %s "), LOG.error(_LE("Unsupported vf device type: %s "),
vf_device_type) vf_device_type)
def _config_vlan_ib_cx3(self, vlan, pf_mlx_dev, dev, hca_port): def _config_vlan_ib_mlnx4(self, vlan, pf_mlx_dev, dev, hca_port):
if vlan == 0: if vlan == 0:
ppkey_idx = self._get_pkey_idx( ppkey_idx = self._get_pkey_idx(
int(DEFAULT_PKEY, 16), pf_mlx_dev, hca_port) int(DEFAULT_PKEY, 16), pf_mlx_dev, hca_port)
@ -372,3 +406,9 @@ class eSwitchHandler(object):
def _config_port_up(self, dev): def _config_port_up(self, dev):
cmd = ['ip', 'link', 'set', dev, 'up'] cmd = ['ip', 'link', 'set', dev, 'up']
command_utils.execute(*cmd) command_utils.execute(*cmd)
def _get_pf_fabric(self, fabric, dev):
fabric_details = self.rm.get_fabric_details(fabric)
for pf_fabric in fabric_details.values():
if dev in pf_fabric['vfs']:
return pf_fabric

41
networking_mlnx/eswitchd/resource_mngr.py
View File

@ -38,7 +38,7 @@ class ResourceManager(object):
pf_mlx_dev) pf_mlx_dev)
vfs = self.discover_devices(pf) vfs = self.discover_devices(pf)
LOG.info(_LI("PF %(pf)s, vfs = %(vf)s"), {'pf': pf, 'vf': vfs}) LOG.info(_LI("PF %(pf)s, vfs = %(vf)s"), {'pf': pf, 'vf': vfs})
self.device_db.set_fabric_devices(fabric, vfs) self.device_db.set_fabric_devices(fabric, pf, vfs)
def scan_attached_devices(self): def scan_attached_devices(self):
devices = [] devices = []
@ -68,8 +68,8 @@ class ResourceManager(object):
vm_ids[dev[0]] = vm_id vm_ids[dev[0]] = vm_id
return devices, vm_ids return devices, vm_ids
def get_fabric_details(self, fabric): def get_fabric_details(self, fabric, pf=None):
return self.device_db.get_fabric_details(fabric) return self.device_db.get_fabric_details(fabric, pf)
def discover_devices(self, pf): def discover_devices(self, pf):
return self.pci_utils.get_vfs_info(pf) return self.pci_utils.get_vfs_info(pf)
@ -98,22 +98,25 @@ class ResourceManager(object):
fabric = self.get_fabric_for_dev(dev) fabric = self.get_fabric_for_dev(dev)
if fabric: if fabric:
fabric_details = self.get_fabric_details(fabric) fabric_details = self.get_fabric_details(fabric)
if fabric_details['pf_device_type'] == \ for pf_fabric_details in fabric_details.values():
constants.CX3_VF_DEVICE_TYPE: if (pf_fabric_details['pf_device_type'] ==
hca_port = fabric_details['hca_port'] constants.MLNX4_VF_DEVICE_TYPE):
pf_mlx_dev = fabric_details['pf_mlx_dev'] hca_port = pf_fabric_details['hca_port']
vf_index = self.pci_utils.get_guid_index(pf_mlx_dev, dev, pf_mlx_dev = pf_fabric_details['pf_mlx_dev']
hca_port) vf_index = self.pci_utils.get_guid_index(
elif fabric_details['pf_device_type'] in \ pf_mlx_dev, dev, hca_port)
[constants.CX4_VF_DEVICE_TYPE, elif (pf_fabric_details['pf_device_type'] ==
constants.CX5_VF_DEVICE_TYPE]: constants.MLNX5_VF_DEVICE_TYPE):
vf_index = fabric_details['vfs'][dev]['vf_num'] if dev in pf_fabric_details['vfs']:
try: vf_index = pf_fabric_details['vfs'][dev]['vf_num']
mac = self.macs_map[fabric][str(vf_index)] else:
devs.append((dev, mac, fabric)) continue
except KeyError: try:
LOG.warning(_LW("Failed to retrieve Hostdev MAC" mac = self.macs_map[fabric][str(vf_index)]
"for dev %s"), dev) devs.append((dev, mac, fabric))
except KeyError:
LOG.warning(_LW("Failed to retrieve Hostdev MAC"
"for dev %s"), dev)
else: else:
LOG.info(_LI("No Fabric defined for device %s"), hostdev) LOG.info(_LI("No Fabric defined for device %s"), hostdev)
return devs return devs

41
networking_mlnx/eswitchd/utils/pci_utils.py
View File

@ -79,12 +79,13 @@ class pciUtils(object):
with open(device_type_file, 'r') as fd: with open(device_type_file, 'r') as fd:
device_type = fd.read() device_type = fd.read()
device_type = device_type.strip(os.linesep) device_type = device_type.strip(os.linesep)
if device_type in constants.CX3_VF_DEVICE_TYPE_LIST: if device_type in constants.MLNX4_VF_DEVICE_TYPE_LIST:
device_vf_type = constants.CX3_VF_DEVICE_TYPE device_vf_type = constants.MLNX4_VF_DEVICE_TYPE
elif device_type in constants.CX4_VF_DEVICE_TYPE_LIST: elif device_type in constants.MLNX5_VF_DEVICE_TYPE_LIST:
device_vf_type = constants.CX4_VF_DEVICE_TYPE device_vf_type = constants.MLNX5_VF_DEVICE_TYPE
elif device_type in constants.CX5_VF_DEVICE_TYPE_LIST: else:
device_vf_type = constants.CX5_VF_DEVICE_TYPE raise Exception(_LE('device type %s is not '
'supported'), device_type)
except IOError: except IOError:
pass pass
return device_vf_type return device_vf_type
@ -170,7 +171,7 @@ class pciUtils(object):
def get_guid_index(self, pf_mlx_dev, dev, hca_port): def get_guid_index(self, pf_mlx_dev, dev, hca_port):
guid_index = None guid_index = None
path = constants.GUID_INDEX_PATH % (pf_mlx_dev, dev, hca_port) path = constants.MLNX4_GUID_INDEX_PATH % (pf_mlx_dev, dev, hca_port)
with open(path) as fd: with open(path) as fd:
guid_index = fd.readline().strip() guid_index = fd.readline().strip()
return guid_index return guid_index
@ -185,23 +186,28 @@ class pciUtils(object):
return return
def get_vfs_macs_ib(self, fabric_details): def get_vfs_macs_ib(self, fabric_details):
if fabric_details['pf_device_type'] == constants.CX3_VF_DEVICE_TYPE: macs_map = {}
return self.get_vfs_macs_ib_cx3(fabric_details) for pf_fabric_details in fabric_details.values():
elif fabric_details['pf_device_type'] == constants.CX4_VF_DEVICE_TYPE: if (pf_fabric_details['pf_device_type'] ==
return self.get_vfs_macs_ib_cx4(fabric_details) constants.MLNX4_VF_DEVICE_TYPE):
macs_map.update(self.get_vfs_macs_ib_mlnx4(pf_fabric_details))
elif (pf_fabric_details['pf_device_type'] ==
constants.MLNX5_VF_DEVICE_TYPE):
macs_map.update(self.get_vfs_macs_ib_mlnx5(pf_fabric_details))
return macs_map
def get_vfs_macs_ib_cx3(self, fabric_details): def get_vfs_macs_ib_mlnx4(self, fabric_details):
hca_port = fabric_details['hca_port'] hca_port = fabric_details['hca_port']
pf_mlx_dev = fabric_details['pf_mlx_dev'] pf_mlx_dev = fabric_details['pf_mlx_dev']
macs_map = {} macs_map = {}
guids_path = constants.ADMIN_GUID_PATH % (pf_mlx_dev, hca_port, guids_path = constants.MLNX4_ADMIN_GUID_PATH % (pf_mlx_dev, hca_port,
'[1-9]*') '[1-9]*')
paths = glob.glob(guids_path) paths = glob.glob(guids_path)
for path in paths: for path in paths:
vf_index = path.split('/')[-1] vf_index = path.split('/')[-1]
with open(path) as f: with open(path) as f:
guid = f.readline().strip() guid = f.readline().strip()
if guid == constants.INVALID_GUID_CX3: if guid == constants.MLNX4_INVALID_GUID:
mac = constants.INVALID_MAC mac = constants.INVALID_MAC
else: else:
head = guid[:6] head = guid[:6]
@ -210,14 +216,15 @@ class pciUtils(object):
macs_map[str(int(vf_index))] = mac macs_map[str(int(vf_index))] = mac
return macs_map return macs_map
def get_vfs_macs_ib_cx4(self, fabric_details): def get_vfs_macs_ib_mlnx5(self, fabric_details):
vfs = fabric_details['vfs'] vfs = fabric_details['vfs']
macs_map = {} macs_map = {}
for vf in vfs.values(): for vf in vfs.values():
vf_num = vf['vf_num'] vf_num = vf['vf_num']
pf_mlx_dev = fabric_details['pf_mlx_dev'] pf_mlx_dev = fabric_details['pf_mlx_dev']
guid_path = constants.CX4_GUID_NODE_PATH % {'module': pf_mlx_dev, guid_path = (
'vf_num': vf_num} constants.MLNX5_GUID_NODE_PATH % {'module': pf_mlx_dev,
'vf_num': vf_num})
with open(guid_path) as f: with open(guid_path) as f:
guid = f.readline().strip() guid = f.readline().strip()
head = guid[:8] head = guid[:8]

2
networking_mlnx/plugins/ml2/drivers/mlnx/agent/mlnx_eswitch_neutron_agent.py
View File

@ -404,7 +404,7 @@ def main():
try: try:
interface_mappings = q_utils.parse_mappings( interface_mappings = q_utils.parse_mappings(
cfg.CONF.ESWITCH.physical_interface_mappings) cfg.CONF.ESWITCH.physical_interface_mappings, unique_keys=False)
except ValueError as e: except ValueError as e:
LOG.error(_LE("Parsing physical_interface_mappings failed: %s. " LOG.error(_LE("Parsing physical_interface_mappings failed: %s. "
"Agent terminated!"), e) "Agent terminated!"), e)

90
networking_mlnx/tests/unit/eswitchd/test_device_db.py
View File

@ -1,90 +0,0 @@
#!/usr/bin/env python
# Copyright 2016 Mellanox Technologies, Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from networking_mlnx.eswitchd.db import device_db
from networking_mlnx.tests import base
class TestDeviceDB(base.TestCase):
def setUp(self):
super(TestDeviceDB, self).setUp()
self.device_db_inst = device_db.DeviceDB()
def _add_fabric(self):
fabric = "fabric"
details = {}
details['pf'] = "pf"
details['pci_id'] = "0000:81:00"
details['hca_port'] = 1
details['fabric_type'] = "fabric_type"
details['pf_mlx_dev'] = "mlx4_0"
self.device_db_inst.add_fabric(fabric, details['pf'],
details['pci_id'], details['hca_port'],
details['fabric_type'], details['pf_mlx_dev'])
return {"fabric": fabric, "details": details}
def test_add_fabric(self):
data = self._add_fabric()
fabric = data["fabric"]
details = data["details"]
self.assertIn(fabric, self.device_db_inst.device_db)
for key in details:
self.assertEqual(details[key],
self.device_db_inst.device_db[fabric][key])
def test_get_fabric_details(self):
data = self._add_fabric()
fabric = data["fabric"]
details = data["details"]
self.assertIn(fabric, self.device_db_inst.device_db)
for key in details:
self.assertEqual(details[key],
self.device_db_inst.get_fabric_details(fabric)[key])
def test_set_fabric_devices(self):
self._add_fabric()
fabric = "fabric"
vfs = {}
pci_id = "0000:81:00.1"
vf_num = 8
vf_device_type = "CX3"
vfs[pci_id] = {'vf_num': vf_num, 'vf_device_type': vf_device_type}
self.device_db_inst.set_fabric_devices(fabric, vfs)
details = self.device_db_inst.get_fabric_details(fabric)
self.assertIn(pci_id, details["vfs"])
self.assertEqual(vf_num, details["vfs"][pci_id]["vf_num"])
self.assertEqual(vf_device_type,
details["vfs"][pci_id]["vf_device_type"])
def test_get_dev_fabric_existent(self):
self._add_fabric()
fabric = "fabric"
vfs = {}
pci_id = "0000:81:00.1"
vf_num = 8
vf_device_type = "CX3"
vfs[pci_id] = {'vf_num': vf_num, 'vf_device_type': vf_device_type}
self.device_db_inst.set_fabric_devices(fabric, vfs)
self.assertEqual(fabric, self.device_db_inst.get_dev_fabric(pci_id))
def test_get_dev_fabric_inexistent(self):
self.assertIsNone(self.device_db_inst.get_dev_fabric("inexistent_dev"))