Cavium SSL加速卡驱动移植到Ubuntu

最新推荐文章于 2024-05-10 18:15:21 发布

zuxi

最新推荐文章于 2024-05-10 18:15:21 发布

阅读量4.8k

点赞数 1

分类专栏： linux 文章标签： cavium

本文链接：https://blog.csdn.net/wangzuxi/article/details/45895297

版权

linux 专栏收录该内容

7 篇文章 1 订阅

订阅专栏

因为cavium ssl加速卡的驱动指定的系统是Fedora，而我们的系统是Ubuntu，所以肯定要移植了，看一下Release_Notes.txt里面的Supported Features：

 Supported Features
 ==================
     System architecture
     -------------------
     - 32/64 bit Intel/AMD platforms
     - Little Endian
     - SMP and Non-SMP

     Operating System and Kernel versions
     ------------------------------------
     OS:
     - Fedora Core 12
     
     Kernel Versions:
     - v2.6.31
     - v2.6.32
     - v2.6.34

可见这个驱动只在v2.6.31、32、34这几个版本的内核上测试过，想在3.0以上版本内核上编译，肯定也要修改一下代码。果不其然，拿它的源码在Ubuntu上make一下报了很多错，这里就不贴编译错误的图了，下面是所做的修改:

1、源码

需要修改的文件有cavium_proc.c、linux_main.c、linux_sysdep.h、Makefile、Makefile.Linux，用Beyond Compare比较如下：

其中cavium_proc.c主要是proc文件系统的修改，因为Linux在3.10内核版本之后对proc文件系统的使用方式和接口有了点变动，所以需要修改，在3.9之前，proc文件系统是这个用的：

static int
proc_write_timeout(struct file *file, const char *buffer,
             unsigned long count, void *data)
{
   char str[count+1], *strend;
   Uint32 timeout;

   CAVIUM_MOD_INC_USE_COUNT;

   if(cavium_copy_in(str, buffer, count))
   {
      cavium_error("cavium_copy_out failed\n");
      return -EFAULT;
   }
   str[count] = '\0';

   timeout = simple_strtoul(str, &strend, 0);
   if (timeout > (2*60*60))
   {
      cavium_error("Timeout too high. The maximum value is 7200\n");
      return -EFAULT;
   }
   cavium_command_timeout = timeout*HZ;

   CAVIUM_MOD_DEC_USE_COUNT;

   return count;
}

static int
proc_read_timeout(char *page, char **start, off_t off, int count,
            int *eof, void *data)
{
   int len = 0;

   CAVIUM_MOD_INC_USE_COUNT;

   len = sprintf(page + len, "%d\n", (cavium_command_timeout/HZ));

   CAVIUM_MOD_DEC_USE_COUNT;

   return len;
}

int cavium_init_proc(void)
{
   int rv = 0;
   /* create directory /proc/cavium */
   cavium_dir = proc_mkdir("cavium", NULL);
   if(cavium_dir == NULL) {
      rv = -ENOMEM;
      goto out;
   }

#if LINUX_VERSION_CODE < KERNEL_VERSION (2,6,30)
   cavium_dir->owner = THIS_MODULE;
#endif

  /* create timeout file */
   timeout_file = create_proc_entry("command_timeout", 0644, cavium_dir);
   if(timeout_file == NULL)
   {
      rv = -ENOMEM;
      goto out;
   }

   timeout_file->data = &cavium_command_timeout;
   timeout_file->read_proc = proc_read_timeout;
   timeout_file->write_proc = proc_write_timeout;

#if LINUX_VERSION_CODE < KERNEL_VERSION (2,6,30)
   timeout_file->owner = THIS_MODULE;

#endif
    ……
}

而在3.10以及3.10之后的内核版本proc文件系统是这么用的：

static ssize_t proc_write_timeout(struct file *file, const char __user *buffer,
                             size_t count, loff_t *ppos)
{
    Uint32 timeout;
    unsigned long val;
    int err;

    CAVIUM_MOD_INC_USE_COUNT;

    err = kstrtoul_from_user(buffer, count, 0, &val);
    if (err)
        return err;

    timeout = val;
    if (timeout > (2*60*60)) {
        cavium_error("Timeout too high. The maximum value is 7200\n");
        return -EFAULT;
    }
    cavium_command_timeout = timeout*HZ;

    CAVIUM_MOD_DEC_USE_COUNT;

    return count;
}

static int proc_read_timeout(struct seq_file *m, void *v)
{
   int len = 0;

   CAVIUM_MOD_INC_USE_COUNT;

   len = seq_printf(m, "%d\n", (cavium_command_timeout/HZ));

   CAVIUM_MOD_DEC_USE_COUNT;

   return len;
}

static struct file_operations cavium_proc_timeout_operations = {
    .open = open_proc_timeout,
    .read = seq_read,
    .llseek = seq_lseek,
    .write = proc_write_timeout,
};

int cavium_init_proc(void)
{
   int rv = 0;
   /* create directory /proc/cavium */
   cavium_dir = proc_mkdir("cavium", NULL);
   if(cavium_dir == NULL) {
      rv = -ENOMEM;
      goto out;
   }

   timeout_file = proc_create("command_timeout", 0644, cavium_dir, &cavium_proc_timeout_operations);
   if(timeout_file == NULL)
   {
      rv = -ENOMEM;
      goto out;
   }
    ……
}

对比一下就会发现接口名变了，参数也变了，所以在cavium_proc.c文件中要做相应的修改，这个文件里面修改的行数还挺多，基本都是修改proc文件系统的使用接口，参考上面的例子就可以了。

文件linux_main.c改得不多，主要是加几个宏定义就可以了，这是因为在内核版本3.7.0之后内核去掉了devinit、devinitdata、devexit_p这几个宏，我们加上这几个宏的定义就可以了：

#if LINUX_VERSION_CODE  > KERNEL_VERSION (3,7,0)
#define __devinit
#define __devinitdata
#define __devexit_p(x) x
#endif

文件linux_sysdep.h改得也不多，主要是几个头文件的问题：

文件driver\linux\Makefile做了只能在2.6内核上编译的限制，要在新版Ubuntu（内核版本3.13）上编译，肯定要去掉这个限制：

文件Makefile.Linux也做了一些修改：

2、编译

经过上面源码的修改，再看看linux_install_driver_pf.txt基本上make clean和make一下就可以了，如果还出错那就再根据出错信息去做相应的修改就可以了。编译还是很简单的。

3、加载

编译成功了的话，在编译目录的bin底下应该有驱动模块：

root@BJDEVR720:~/work/cavium/Cavium_Networks/NITROX-SDK# ls -l bin/
total 0
lrwxrwxrwx 1 root root 69 Apr 14 14:29 csp1_init -> /root/work/cavium/Cavium_Networks/NITROX-SDK/utils/csp1_init*
lrwxrwxrwx 1 root root 71 Apr 14 14:29 init_nitrox -> /root/work/cavium/Cavium_Networks/NITROX-SDK/utils/init_nitrox*
lrwxrwxrwx 1 root root 77 Apr 14 14:29 pkp_drv.ko -> /root/work/cavium/Cavium_Networks/NITROX-SDK/driver/linux/pkp_drv.ko
      然后在bin目录底下执行命令：./init_nitrox ssl就可以加载驱动模块了，如果不出意外，基本上就完事了，但在Ubuntu一些版本上总是出现硬件无法分配资源以及驱动初始化失败的问题：
root@BJDEVR720:~/work/cavium/Cavium_Networks/NITROX-SDK/bin# lspci -vvv -s 05:00.0                         
05:00.0 Network and computing encryption device: Cavium Networks Device 0011 (rev 01)
        Subsystem: Cavium Networks Device 0011
        Control: I/O- Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
        Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
        Latency: 0, Cache Line Size: 64 bytes
        Interrupt: pin A routed to IRQ 48
        Region 0: Memory at <ignored> (64-bit, prefetchable)
        Capabilities: [40] Power Management version 3
                Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0-,D1-,D2-,D3hot-,D3cold-)
                Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
        Capabilities: [50] MSI: Enable- Count=1/1 Maskable+ 64bit+
                Address: 0000000000000000  Data: 0000
                Masking: 00000000  Pending: 00000000
        Capabilities: [70] Express (v2) Endpoint, MSI 00
                DevCap: MaxPayload 256 bytes, PhantFunc 0, Latency L0s <1us, L1 <8us
                        ExtTag+ AttnBtn- AttnInd- PwrInd- RBE+ FLReset+
                DevCtl: Report errors: Correctable- Non-Fatal+ Fatal+ Unsupported+
                        RlxdOrd+ ExtTag- PhantFunc- AuxPwr- NoSnoop+ FLReset-
                        MaxPayload 256 bytes, MaxReadReq 512 bytes
                DevSta: CorrErr- UncorrErr- FatalErr- UnsuppReq- AuxPwr- TransPend-
                LnkCap: Port #0, Speed 5GT/s, Width x16, ASPM L0s L1, Exit Latency L0s <1us, L1 <64us
                        ClockPM- Surprise- LLActRep- BwNot-
                LnkCtl: ASPM Disabled; RCB 64 bytes Disabled- CommClk+
                        ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
                LnkSta: Speed 5GT/s, Width x8, TrErr- Train- SlotClk+ DLActive- BWMgmt- ABWMgmt-
                DevCap2: Completion Timeout: Not Supported, TimeoutDis+, LTR-, OBFF Not Supported
                DevCtl2: Completion Timeout: 50us to 50ms, TimeoutDis+, LTR-, OBFF Disabled
                LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-
                         Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
                         Compliance De-emphasis: -6dB
                LnkSta2: Current De-emphasis Level: -6dB, EqualizationComplete-, EqualizationPhase1-
                         EqualizationPhase2-, EqualizationPhase3-, LinkEqualizationRequest-
        Capabilities: [b0] MSI-X: Enable- Count=19 Masked-
                Vector table: BAR=0 offset=00001000
                PBA: BAR=0 offset=00002000
        Capabilities: [100 v2] Advanced Error Reporting
                UESta:  DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
                UEMsk:  DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt+ UnxCmplt+ RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
                UESvrt: DLP+ SDES+ TLP+ FCP+ CmpltTO+ CmpltAbrt- UnxCmplt- RxOF+ MalfTLP+ ECRC+ UnsupReq- ACSViol-
                CESta:  RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr-
                CEMsk:  RxErr+ BadTLP+ BadDLLP+ Rollover+ Timeout+ NonFatalErr+
                AERCap: First Error Pointer: 00, GenCap+ CGenEn- ChkCap+ ChkEn-
        Capabilities: [140 v1] Alternative Routing-ID Interpretation (ARI)
                ARICap: MFVC- ACS-, Next Function: 0
                ARICtl: MFVC- ACS-, Function Group: 0
        Capabilities: [150 v1] Single Root I/O Virtualization (SR-IOV)
                IOVCap: Migration-, Interrupt Message Number: 000
                IOVCtl: Enable- Migration- Interrupt- MSE- ARIHierarchy+
                IOVSta: Migration-
                Initial VFs: 64, Total VFs: 64, Number of VFs: 0, Function Dependency Link: 00
                VF offset: 1, stride: 1, Device ID: 0011
                Supported Page Size: 00000553, System Page Size: 00000001
                Region 0: Memory at 0000000000000000 (64-bit, prefetchable)
                VF Migration: offset: 00000000, BIR: 0

root@BJDEVR720:~/work/cavium/Cavium_Networks/NITROX-SDK/bin# dmesg 
[  239.840522] VFCOUNT 0
[  239.840604]  requested mem region for bar 0 cannot be allocated
[  239.840610]  Cavium Init failed for device 
[  239.840644] pkp: probe of 0000:05:00.0 failed with error -12
[  239.840697]  requested mem region for bar 0 cannot be allocated
[  239.840702]  Cavium Init failed for device 
[  239.840719] pkp: probe of 0000:43:00.0 failed with error -12
root@BJDEVR720:~/work/cavium/Cavium_Networks/NITROX-SDK/bin#

这个问题困扰我很长时间，看内核pci驱动初始化代码也没找出是啥问题，后来峰哥在https://bugzilla.kernel.org/show_bug.cgi?id=10461上看到人家只要加一个内核参数就能解决问题了，在我们系统试了一下果然也能解决掉这个问题，加的内核启动参数是pci=nocrs，这个参数对应到代码是在arch/x86/pci/common.c和arch/x86/pci/acpi.c文件中有用到，最后体现在acpi.c的函数pci_acpi_scan_root中：

如果加了内核启动参数pci=nocrs的话，应该执行的是上图else分支，看样子是先释放一些资源再添加一些资源，具体原因就没有往更深层次去挖掘了，因为这个需要熟悉pci一整套流程和玩法才好理解，这个还是比较复杂的，目前功力尚未达到这个境界，先这么着吧。添加内核启动参数只需要在/boot/grub/grub.cfg加一下就可以了：

修改完重启之后加载驱动：

root@BJDEVR720:~/work/cavium/Cavium_Networks/NITROX-SDK/bin# ./init_nitrox ssl
insmod ./pkp_drv.ko ssl=0 
./csp1_init ssl=0 
File: ../microcode/boot_mc2_n3.out
        Version = CN35x-MC-Boot-0001
        Code length = 14060     Data length = 200
0: name=../microcode/boot_mc2_n3.out, index=0, core=0
File: ../microcode/main_ssl2_n3.out
        Version = CNN35x-MC-SSL-0008
        Code length = 59620     Data length = 320
1: name=../microcode/main_ssl2_n3.out, index=1, core=0
Microcode Load Succeed
root@BJDEVR720:~/work/cavium/Cavium_Networks/NITROX-SDK/bin# dmesg 
[ 4151.317917] VFCOUNT 0
[ 4151.318649] Nitrox3 found at Bus 5 Slot 0
[ 4151.318827] Using memory-mapped bar for device 0x177D:0x11
[ 4151.320505] bar 0: d9200000
[ 4151.331446] SERDES_TUNING register value is already set: 0x458cc58
[ 4151.334542] detect_n3_sku: Invalid zip mask 0x0000000f
[ 4151.431688] part number=CNN35XX has 16 cores
[ 4151.431691] Finished Initializing this device
[ 4151.431693] Total Number of Nitrox Devices: 1
[ 4151.432981] Loaded Cavium Driver --- 5.03-
[ 4152.456226] got boot microcode
[ 4152.456243] Code type = 02, code size = 36ec, data size = c8
[ 4152.456246] got mainline microcode
[ 4152.456269] Code type = 01, code size = e8e4, data size = 140
[ 4152.812956] UEN = 0xffff
[ 4152.812960] Final Core Mask = 0x0000ffff
[ 4152.812962] Loaded Boot microcode
[ 4154.315289] Loaded Main microcode
[ 4154.350374] Loading of Microcodes successful
[ 4154.370396] 
[ 4154.370396] Core Frequency 500 MHz
root@BJDEVR720:~/work/cavium/Cavium_Networks/NITROX-SDK/bin#

从加载结果和dmesg调试信息可见已经正常加载驱动了，再lspci一下看看：

root@rdserver5:~/work/cavium/Cavium_Networks/NITROX-SDK/bin# lspci -vvv -s 05:00.0
05:00.0 Network and computing encryption device: Cavium Networks Device 0011 (rev 01)
        Subsystem: Cavium Networks Device 0011
        Control: I/O- Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
        Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
        Latency: 0, Cache Line Size: 64 bytes
        Interrupt: pin A routed to IRQ 48
        Region 0: Memory at d9200000 (64-bit, prefetchable) [size=1M]
        Capabilities: [40] Power Management version 3
                Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0-,D1-,D2-,D3hot-,D3cold-)
                Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
        Capabilities: [50] MSI: Enable- Count=1/1 Maskable+ 64bit+
                Address: 0000000000000000  Data: 0000
                Masking: 00000000  Pending: 00000000
        Capabilities: [70] Express (v2) Endpoint, MSI 00
                DevCap: MaxPayload 256 bytes, PhantFunc 0, Latency L0s <1us, L1 <8us
                        ExtTag+ AttnBtn- AttnInd- PwrInd- RBE+ FLReset+
                DevCtl: Report errors: Correctable- Non-Fatal+ Fatal+ Unsupported+
                        RlxdOrd+ ExtTag+ PhantFunc- AuxPwr- NoSnoop+ FLReset-
                        MaxPayload 256 bytes, MaxReadReq 2048 bytes
                DevSta: CorrErr- UncorrErr- FatalErr- UnsuppReq- AuxPwr- TransPend-
                LnkCap: Port #0, Speed 5GT/s, Width x16, ASPM L0s L1, Latency L0 <1us, L1 <64us
                        ClockPM- Surprise- LLActRep- BwNot-
                LnkCtl: ASPM Disabled; RCB 64 bytes Disabled- Retrain- CommClk+
                        ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
                LnkSta: Speed 5GT/s, Width x8, TrErr- Train- SlotClk+ DLActive- BWMgmt- ABWMgmt-
                DevCap2: Completion Timeout: Not Supported, TimeoutDis+
                DevCtl2: Completion Timeout: 50us to 50ms, TimeoutDis+
                LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-, Selectable De-emphasis: -6dB
                         Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
                         Compliance De-emphasis: -6dB
                LnkSta2: Current De-emphasis Level: -6dB
        Capabilities: [b0] MSI-X: Enable- Count=19 Masked-
                Vector table: BAR=0 offset=00001000
                PBA: BAR=0 offset=00002000
        Capabilities: [100 v2] Advanced Error Reporting
                UESta:  DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
                UEMsk:  DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt+ UnxCmplt+ RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
                UESvrt: DLP+ SDES+ TLP+ FCP+ CmpltTO+ CmpltAbrt- UnxCmplt- RxOF+ MalfTLP+ ECRC+ UnsupReq- ACSViol-
                CESta:  RxErr+ BadTLP- BadDLLP- Rollover+ Timeout+ NonFatalErr-
                CEMsk:  RxErr+ BadTLP+ BadDLLP+ Rollover+ Timeout+ NonFatalErr+
                AERCap: First Error Pointer: 00, GenCap+ CGenEn- ChkCap+ ChkEn-
        Capabilities: [140 v1] Alternative Routing-ID Interpretation (ARI)
                ARICap: MFVC- ACS-, Next Function: 0
                ARICtl: MFVC- ACS-, Function Group: 0
        Capabilities: [150 v1] Single Root I/O Virtualization (SR-IOV)
                IOVCap: Migration-, Interrupt Message Number: 000
                IOVCtl: Enable- Migration- Interrupt- MSE- ARIHierarchy+
                IOVSta: Migration-
                Initial VFs: 64, Total VFs: 64, Number of VFs: 0, Function Dependency Link: 00
                VF offset: 1, stride: 1, Device ID: 0011
                Supported Page Size: 00000553, System Page Size: 00000001
                Region 0: Memory at 00000000c0000000 (64-bit, prefetchable)
                VF Migration: offset: 00000000, BIR: 0
        Kernel driver in use: pkp

可以看出两个Region所在行是正常值而不是ignored了。

整个移植过程完成。

zuxi

关注

1
点赞
踩
5

收藏

觉得还不错? 一键收藏
9
评论
Cavium SSL加速卡驱动移植到Ubuntu

因为cavium ssl加速卡的驱动指定的系统是Fedora，而我们的系统是Ubuntu，所以肯定要移植了，看一下Release_Notes.txt里面的Supported Features： Supported Features ================== System architecture ------------------- - 32/64
复制链接

扫一扫