我有一台Supermicro 2027TR-HTFRF服务器,我用它作为xen服务器.
我想要做的是为Connectx-3卡运行SR-IOV,以便我可以将Infiniband暴露给虚拟机.
按照这个方法,我设法让I350以太网暴露VF.
添加/etc/modprobe.d/mlx4_core.conf包含
options max_vfs=4
尽管这是mlx4_core模块的有效参数,但没有为mellanox设备创建虚函数.即
我得到的错误是
mlx4_core: `4' invalid for parameter `max_vfs'
我也尝试了1值,我得到了类似的错误
mlx4_core: `1' invalid for parameter `max_vfs'
从lspci -v输出
06:00.0 Network controller: Mellanox Technologies MT27500 Family [ConnectX-3]
Subsystem: Mellanox Technologies Device 0051
Flags: bus master,fast devsel,latency 0,IRQ 11
Memory at df900000 (64-bit,non-prefetchable) [size=1M]
Memory at de000000 (64-bit,prefetchable) [size=8M]
Capabilities: [40] Power Management version 3
Capabilities: [48] Vital Product Data
Capabilities: [9c] MSI-X: Enable- Count=128 Masked-
Capabilities: [60] Express Endpoint,MSI 00
Capabilities: [100] Alternative Routing-ID Interpretation (ARI)
Capabilities: [148] Device Serial Number 00-25-90-ff-ff-17-57-24
Capabilities: [18c] #19
Kernel modules: mlx4_core
所以似乎SR-IOV不可用.
然后我发现了这个帖子：
http://community.mellanox.com/thread/1073
但我正在使用ubuntu 12.04.
接下来我使用apt-get安装了mstflint.
然后我确定了它是什么类型的卡.
mstflint -d 06：00.0 q
不幸的是,mstflint崩溃了.
Warning: memory access to device 06:00.0 Failed: Input/output error. Warning: Fallback on IO: much slower,and unsafe if device in use. *** buffer overflow detected ***: mstflint terminated ======= Backtrace: ========= /lib/x86_64-linux-gnu/libc.so.6(__fortify_fail+0x37)[0x7f66d91ac817]
一些strace输出：
strace mstflint -d 06:00.0 q
...
open("/proc/bus/pci/06/00.0",O_RDWR|O_DSYNC) = 3
ioctl(3,IIOCNETDIF,0xdf900000) = 0
mmap(NULL,1048576,PROT_READ|PROT_WRITE,MAP_SHARED,3,0xdf900000) = 0x7f0e0a200000
munmap(0x7f0e0a200000,65536) = 0
close(3) = 0
open("/dev/mem",O_RDWR|O_DSYNC) = 3
mmap(NULL,0xdf900000) = 0x7f0e092f2000
munmap(0x7f0e092f2000,65536) = 0
close(3) = 0
write(2,"Warning: memory access to device"...,136Warning: memory access to device 06:00.0 Failed: Input/output error.
Warning: Fallback on IO: much slower,and unsafe if device in use.
) = 136
open("/sys/bus/pci/devices/0000:06:00.0/config",O_RDWR|O_DSYNC) = 3
pwrite(3,"\375\374\373\372",4,88) = 4
pread(3,88) = 4
close(3) = 0
open("/dev/tty",O_RDWR|O_NOCTTY|O_NONBLOCK) = 3
writev(3,[{"*** ",4},{"buffer overflow detected",24},{" ***: ",6},{"mstflint",8},{" terminated\n",12}],5*** buffer overflow detected ***: mstflint terminated
) = 54
mmap(NULL,4096,MAP_PRIVATE|MAP_ANONYMOUS,-1,0) = 0x7f0e0a3f9000
write(3,"======= Backtrace: =========\n",29======= Backtrace: =========
) = 29
writev(3,[{"/lib/x86_64-linux-gnu/libc.so.6",31},{"(",1},{"__fortify_fail",14},{"+0x",3},{"37",2},{")",{"[0x",{"7f0e094fc817",12},{"]\n",2}],9/lib/x86_64-linux-gnu/libc.so.6(__fortify_fail+0x37)[0x7f0e094fc817]
) = 69
writev(3,{"109710",{"7f0e094fb710",8/lib/x86_64-linux-gnu/libc.so.6(+0x109710)[0x7f0e094fb710]
) = 59
writev(3,{"108b79",{"7f0e094fab79",8/lib/x86_64-linux-gnu/libc.so.6(+0x108b79)[0x7f0e094fab79]
) = 59
writev(3,{"_IO_default_xsputn",18},{"dd",{"7f0e0946e13d",9/lib/x86_64-linux-gnu/libc.so.6(_IO_default_xsputn+0xdd)[0x7f0e0946e13d]
) = 73
writev(3,{"_IO_vfprintf",{"1ae7",{"7f0e0943c4a7",9/lib/x86_64-linux-gnu/libc.so.6(_IO_vfprintf+0x1ae7)[0x7f0e0943c4a7]
) = 69
writev(3,{"__vsprintf_chk",{"94",{"7f0e094fac14",9/lib/x86_64-linux-gnu/libc.so.6(__vsprintf_chk+0x94)[0x7f0e094fac14]
) = 69
writev(3,{"__sprintf_chk",13},{"7d",{"7f0e094fab5d",9/lib/x86_64-linux-gnu/libc.so.6(__sprintf_chk+0x7d)[0x7f0e094fab5d]
) = 68
writev(3,[{"mstflint",{"40de46",4mstflint[0x40de46]
) = 19
writev(3,{"40ee56",4mstflint[0x40ee56]
) = 19
writev(3,{"406475",4mstflint[0x406475]
) = 19
writev(3,{"408d00",4mstflint[0x408d00]
) = 19
writev(3,{"__libc_start_main",17},{"ed",{"7f0e0941376d",9/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xed)[0x7f0e0941376d]
) = 72
writev(3,{"401af9",4mstflint[0x401af9]
) = 19
write(3,"======= Memory map: ========\n",29======= Memory map: ========
) = 29
open("/proc/self/maps",O_RDONLY) = 4
read(4,"00400000-0041a000 r-xp 00000000 "...,1024) = 1024
write(3,102400400000-0041a000 r-xp 00000000 08:01 220 /usr/bin/mstflint
00619000-0061a000 r--p 00019000 08:01 220 /usr/bin/mstflint
0061a000-0061b000 rw-p 0001a000 08:01 220 /usr/bin/mstflint
024a3000-024c4000 rw-p 00000000 00:00 0 [heap]
7f0e09302000-7f0e093f2000 rw-s df910000 00:05 1028 /dev/mem
7f0e093f2000-7f0e095a7000 r-xp 00000000 08:01 135350 /lib/x86_64-linux-gnu/libc-2.15.so
7f0e095a7000-7f0e097a6000 ---p 001b5000 08:01 135350 /lib/x86_64-linux-gnu/libc-2.15.so
7f0e097a6000-7f0e097aa000 r--p 001b4000 08:01 135350 /lib/x86_64-linux-gnu/libc-2.15.so
7f0e097aa000-7f0e097ac000 rw-p 001b8000 08:01 135350 /lib/x86_64-linux-gnu/libc-2.15.so
7f0e097ac000-7f0e097b1000 rw-p 00000000 00:00 0
7f0e097b1000-7f0e097c6000 r-xp 00000000 08:01 131116 /lib/x86_64-linux-gnu/libgcc_s.so.1) = 1024
read(4,"\n7f0e097c6000-7f0e099c5000 ---p "...,1024
7f0e097c6000-7f0e099c5000 ---p 00015000 08:01 131116 /lib/x86_64-linux-gnu/libgcc_s.so.1
7f0e099c5000-7f0e099c6000 r--p 00014000 08:01 131116 /lib/x86_64-linux-gnu/libgcc_s.so.1
7f0e099c6000-7f0e099c7000 rw-p 00015000 08:01 131116 /lib/x86_64-linux-gnu/libgcc_s.so.1
7f0e099c7000-7f0e09ac2000 r-xp 00000000 08:01 135361 /lib/x86_64-linux-gnu/libm-2.15.so
7f0e09ac2000-7f0e09cc1000 ---p 000fb000 08:01 135361 /lib/x86_64-linux-gnu/libm-2.15.so
7f0e09cc1000-7f0e09cc2000 r--p 000fa000 08:01 135361 /lib/x86_64-linux-gnu/libm-2.15.so
7f0e09cc2000-7f0e09cc3000 rw-p 000fb000 08:01 135361 /lib/x86_64-linux-gnu/libm-2.15.so
7f0e09cc3000-7f0e09da5000 r-xp 00000000 08:01 30579 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.16
7f0e09da5000-7f0e09fa4000 ---p 000e2000 08:01 30579 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.16
7f0e09fa4000-7f0e09fac000 ) = 1024
read(4,"r--p 000e1000 08:01 30579 "...,1024r--p 000e1000 08:01 30579 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.16
7f0e09fac000-7f0e09fae000 rw-p 000e9000 08:01 30579 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.16
7f0e09fae000-7f0e09fc3000 rw-p 00000000 00:00 0
7f0e09fc3000-7f0e09fd9000 r-xp 00000000 08:01 131300 /lib/x86_64-linux-gnu/libz.so.1.2.3.4
7f0e09fd9000-7f0e0a1d8000 ---p 00016000 08:01 131300 /lib/x86_64-linux-gnu/libz.so.1.2.3.4
7f0e0a1d8000-7f0e0a1d9000 r--p 00015000 08:01 131300 /lib/x86_64-linux-gnu/libz.so.1.2.3.4
7f0e0a1d9000-7f0e0a1da000 rw-p 00016000 08:01 131300 /lib/x86_64-linux-gnu/libz.so.1.2.3.4
7f0e0a1da000-7f0e0a1fc000 r-xp 00000000 08:01 135367 /lib/x86_64-linux-gnu/ld-2.15.so
7f0e0a210000-7f0e0a300000 rw-s df910000 00:03 4026533088 /proc/bus/pci/06/00.0
7f0e0a300000-7f0e0a3f0000 rw-s df910000 00:0e 19851 /sys/devices/pci0000:00/0000:00:03.2/0000:06:) = 1024
read(4,"00.0/resource0\n7f0e0a3f0000-7f0e"...,1024) = 570
write(3,57000.0/resource0
7f0e0a3f0000-7f0e0a3f5000 rw-p 00000000 00:00 0
7f0e0a3f9000-7f0e0a3fc000 rw-p 00000000 00:00 0
7f0e0a3fc000-7f0e0a3fd000 r--p 00022000 08:01 135367 /lib/x86_64-linux-gnu/ld-2.15.so
7f0e0a3fd000-7f0e0a3ff000 rw-p 00023000 08:01 135367 /lib/x86_64-linux-gnu/ld-2.15.so
7fffdd53f000-7fffdd560000 rw-p 00000000 00:00 0 [stack]
7fffdd5ff000-7fffdd600000 r-xp 00000000 00:00 0 [vdso]
ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]
) = 570
read(4,"",1024) = 0
close(4) = 0
rt_sigprocmask(SIG_UNBLOCK,[ABRT],NULL,8) = 0
gettid() = 4122
tgkill(4122,4122,SIGABRT) = 0
--- SIGABRT (Aborted) @ 0 (0) ---
+++ killed by SIGABRT (core dumped) +++
Aborted (core dumped)
更新：
我设法从SuperMicro获得了固件更新,该更新已启用SR-IOV.
另外,我需要将选项max_vfs =设置为选项num_vfs =.似乎命名参数略有不同.这足以让它在Ubuntu下工作.
安装我使用此方法的固件：http://www.panticz.de/mellanox
但是,在调用mst start之前,我必须在每个模块上调用insmod. Modprobe不起作用.
首先检查明显的东西：
>在BIOS设置中,确保已启用Intel VT-d和Intel I / OAT.
>在另一个PCI插槽中尝试使用该卡.
> Contact the package maintainer报告工具中的错误.
