In the meantime I took one of the servers out of production and installed a clean version of ESX 6.0.
If you would like to use the infiniband driver package then remove the inbox ethernet drivers with
# esxcli software vib remove -n=nmlx4-core -n=nmlx4-en -n=nmlx4-rdma
and install the last driver package for 5.*
# esxcli software vib install -d /MLNX-OFED-ESX-1.8.2.4-10EM-500.0.0.472560.zip --no-sig-check
Et voila
[root@zd-esx-s03:~] esxcfg-nics -l
Name PCI Driver Link Speed Duplex MAC Address MTU Description
vmnic0 0000:0b:00.0 igb Up 1000Mbps Full 00:19:99:9d:36:4c 1500 Intel Corporation 82575EB Gigabit Network Connection
vmnic1 0000:0b:00.1 igb Up 1000Mbps Full 00:19:99:9d:36:4d 1500 Intel Corporation 82575EB Gigabit Network Connection
vmnic3 0000:04:00.0 e1000e Up 1000Mbps Full 00:1b:21:28:86:a5 1500 Intel Corporation 82571EB Gigabit Ethernet Controller (Copper)
vmnic4 0000:04:00.1 e1000e Up 1000Mbps Full 00:1b:21:28:86:a4 1500 Intel Corporation 82571EB Gigabit Ethernet Controller (Copper)
vmnic5 0000:05:00.0 e1000e Up 1000Mbps Full 00:1b:21:28:86:a7 1500 Intel Corporation 82571EB Gigabit Ethernet Controller (Copper)
vmnic6 0000:05:00.1 e1000e Up 1000Mbps Full 00:1b:21:28:86:a6 1500 Intel Corporation 82571EB Gigabit Ethernet Controller (Copper)
vmnic_ib0 0000:06:00.0 ib_ipoib Up 20000Mbps Full 00:02:c9:ea:ea:d1 2044 Mellanox Technologies MT27500 Family [ConnectX-3]
vmnic_ib1 0000:06:00.0 ib_ipoib Up 20000Mbps Full 00:02:c9:ea:ea:d2 2044 Mellanox Technologies MT27500 Family [ConnectX-3]
[root@zd-esx-s03:~] vmkping 192.168.61.54
PING 192.168.61.54 (192.168.61.54): 56 data bytes
64 bytes from 192.168.61.54: icmp_seq=0 ttl=255 time=0.279 ms