目录
调用dpdk-devbind.py脚本来绑定PCI设备igb_uio驱动
整体加载思路
1、加载IGB_UIO驱动；
2、加载VFIO模块；
3、配置大页内存；
4、set_numa_pages；
5、给PCIE网卡设备号加载IGB_UIO驱动；
6、加载载KNI模块，用于跑DPDK流的同时，并行跑标准内核协议.
配置步骤
全局变量配置
#配置DPDK安装路径
export RTE_SDK=$PWD
#定义PCIE网络端口设备号
export PCI_DEVICE0=0000:b3:00.0
export PCI_DEVICE1=0000:b3:00.1
#配置大页内存个数，结合当前大页内存个数来自定义
export Pages=20
#DPDK目标环境目录
export RTE_TARGET=x86_64-native-linux-gcc
echo "-------------------------------------------------------------------------- ----"
echo " RTE_SDK exported as $RTE_SDK"
echo "-------------------------------------------------------------------------- ----"
#获取大页内存大小
HUGEPGSZ=`cat /proc/meminfo | grep Hugepagesize | cut -d : -f 2 | tr -d ' '`
调用流程编写
#加载IGB_UIO驱动
echo "LOAD IGB_UIO"
load_igb_uio
#加载VFIO模块
echo "LOAD VFIO"
load_vfio_module
#配置大页内存
echo "SET HUGE PAGES"
set_numa_pages
#给PCIE网卡设备号加载IGB_UIO
echo "BIND DEVICE -> $PCI_DEVICE0 and $PCI_DEVICE1"
echo "BIND DEVICE -> $PCI_DEVICE1 and $PCI_DEVICE1"
bind_devices_to_igb_uio
#加载载KNI模块，用于跑DPDK流的同时，并行跑标准内核协议
echo "LOAD KNI"
load_kni_module
加载VFIO模块
load_vfio_module()
{
remove_vfio_module
VFIO_PATH="kernel/drivers/vfio/pci/vfio-pci.ko"
echo "Loading VFIO module"
/sbin/lsmod | grep -s vfio_pci > /dev/null
if [ $? -ne 0 ] ; then
if [ -f /lib/modules/$(uname -r)/$VFIO_PATH ] ; then
sudo /sbin/modprobe vfio-pci
fi
fi
# make sure regular users can read /dev/vfio
echo "chmod /dev/vfio"
sudo chmod a+x /dev/vfio
if [ $? -ne 0 ] ; then
echo "FAIL"
quit
fi
echo "OK"
# check if /dev/vfio/vfio exists - that way we
# know we either loaded the module, or it was
# compiled into the kernel
if [ ! -e /dev/vfio/vfio ] ; then
echo "## ERROR: VFIO not found!"
fi
}
删除VFIO模块
remove_vfio_module()
{
echo "Unloading any existing VFIO module"
/sbin/lsmod | grep -s vfio > /dev/null
if [ $? -eq 0 ] ; then
sudo /sbin/rmmod vfio-pci
sudo /sbin/rmmod vfio_iommu_type1
sudo /sbin/rmmod vfio
fi
}
加载KNI模块
load_kni_module()
{
# Check that the KNI module is already built.
if [ ! -f $RTE_SDK/$RTE_TARGET/kmod/rte_kni.ko ];then
echo "## ERROR: Target does not have the DPDK KNI Module."
echo " To fix, please try to rebuild target."
return
fi
# Unload existing version if present.
remove_kni_module
# Now try load the KNI module.
echo "Loading DPDK KNI module"
sudo /sbin/insmod $RTE_SDK/$RTE_TARGET/kmod/rte_kni.ko
if [ $? -ne 0 ] ; then
echo "## ERROR: Could not load kmod/rte_kni.ko."
quit
fi
}
卸载KNI模块
remove_kni_module()
{
echo "Unloading any existing DPDK KNI module"
/sbin/lsmod | grep -s rte_kni > /dev/null
if [ $? -eq 0 ] ; then
sudo /sbin/rmmod rte_kni
fi
}
创建大页内存
# Creates hugepages on specific NUMA nodes.
set_numa_pages()
{
clear_huge_pages
#echo ""
#echo " Input the number of ${HUGEPGSZ} hugepages for each node"
#echo " Example: to have 128MB of hugepages available per node in a 2MB huge page system,"
#echo " enter '64' to reserve 64 * 2MB pages on each node"
#echo > .echo_tmp
for d in /sys/kernel/mm ; do
node=$(basename $d)
# echo -n "Number of pages for $node: $Pages"
echo "Number of pages for $node: $Pages"
# read Pages
echo "echo $Pages > $d/hugepages/hugepages-${HUGEPGSZ}/nr_hugepages" >> .echo_tmp
done
echo "Reserving hugepages"
sudo sh .echo_tmp
rm -f .echo_tmp
create_mnt_huge
}
创建大页内存文件系统
# Creates hugepage filesystem.
create_mnt_huge()
{
echo "Creating /mnt/huge and mounting as hugetlbfs"
sudo mkdir -p /mnt/huge
grep -s '/mnt/huge' /proc/mounts > /dev/null
if [ $? -ne 0 ] ; then
sudo mount -t hugetlbfs nodev /mnt/huge
fi
}
删除遗留的大页内存
# Removes all reserved hugepages.
clear_huge_pages()
{
echo > .echo_tmp
for d in /sys/kernel/mm ; do
echo "echo 0 > $d/hugepages/hugepages-${HUGEPGSZ}/nr_hugepages" >> .echo_tmp
done
echo "Removing currently reserved hugepages"
sudo sh .echo_tmp
rm -f .echo_tmp
remove_mnt_huge
}
卸载大页文件系统
remove_mnt_huge()
{
echo "Unmounting /mnt/huge and removing directory"
grep -s '/mnt/huge' /proc/mounts > /dev/null
if [ $? -eq 0 ] ; then
sudo umount /mnt/huge
fi
if [ -d /mnt/huge ] ; then
sudo rm -R /mnt/huge
fi
}
调用dpdk-devbind.py脚本来绑定PCI设备igb_uio驱动
bind_devices_to_igb_uio()
{
if [ -d /sys/module/igb_uio ]; then
#${RTE_SDK}/usertools/dpdk-devbind.py --status
#echo ""
#echo -n "Enter PCI address of device to bind to IGB UIO driver: "
#read PCI_DEVICE
sudo ${RTE_SDK}/usertools/dpdk-devbind.py -b igb_uio $PCI_DEVICE0 && echo "$PCI_DEVICE0 Bind OK"
sudo ${RTE_SDK}/usertools/dpdk-devbind.py -b igb_uio $PCI_DEVICE1 && echo "$PCI_DEVICE1 Bind OK"
else
echo "# Please load the 'igb_uio' kernel module before querying or "
echo "# adjusting device bindings"
fi
}
加载igb_uio.ko驱动.
load_igb_uio()
{
#检查igb_uio驱动是否存在，不存在则退出，输出错误日志
if [ ! -f $RTE_SDK/$RTE_TARGET/kmod/igb_uio.ko ];then
echo "## ERROR: Target does not have the DPDK UIO Kernel Module. "
echo " To fix, please try to rebuild target."
return
fi
#在加载新的igb_uio前，先卸载旧的igb_uio驱动
unload_igb_uio
#查找UIO驱动是否存在，
/sbin/lsmod | grep -s uio > /dev/null
if [ $? -ne 0 ] ; then
#输出UIO驱动信息
modinfo uio 1>/dev/null 2>/dev/null
if [ $? -eq 0 ]; then
#加载UIO模块
echo "Loading uio module"
sudo /sbin/modprobe uio
fi
fi
# UIO may be compiled into kernel, so it may not be an error if it can't
# be loaded.
#加载载igb_uio驱动
echo "Loading DPDK UIO module"
sudo /sbin/insmod $RTE_SDK/$RTE_TARGET/kmod/igb_uio.ko
if [ $? -ne 0 ] ; then
echo "## ERROR: Could not load kmod/igb_uio.ko."
quit
fi
}
卸载 igb_uio.ko 驱动
unload_igb_uio()
{
echo "Unloading any existing DPDK UIO module"
/sbin/lsmod | grep -s igb_uio > /dev/null
if [ $? -eq 0 ] ; then
sudo /sbin/rmmod igb_uio 1>/dev/null 2>/dev/null
fi
}
617
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
