dpdk 内存管理

dpdk 使用 Linux 提供的获取大页、页表、numa 节点表、mmap 功能重新组织内存。

• 大页： 最多可以同时存在 3 种大页。一般只是用 2M 的大页。linux 中获取大页相关信息是通过访问解析 /sys/kernel/mm/hugepages、/proc/meminfo、/proc/mounts 等内容得到的。

• 页表： linux 中每个进程的页表对应的文件是 /proc/pid/pagemap，本进程的页表是 /proc/self/pagemap。

• numa 节点表： linux 中每个物理页所在的 numa socket 表对应的文件是 /proc/pid/numa_maps，本进程的 numa socket 表是 /proc/self/numa_maps。

• 大页内存：Hugepages详解

[root@localhost simple_mp]#   ./build/simple_mp -l 126-127   --proc-type=primary
EAL: Detected 128 lcore(s)
EAL: Detected 4 NUMA nodes
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'PA'
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL: PCI device 0000:05:00.0 on NUMA socket 0
EAL:   probe driver: 19e5:200 net_hinic

[root@localhost simple_mp]# ps -elf | grep simple
0 R root      15154 124128 96  80   0 - 8389397 -    05:27 pts/1    00:00:09 ./build/simple_mp -l 126-127 --proc-type=primary
0 S root      15168   7504  0  80   0 -  1729 pipe_w 05:27 pts/2    00:00:00 grep --color=auto simple
[root@localhost simple_mp]# cat /proc/15154/numa_maps
00400000 default file=/data1/dpdk-19.11/examples/multi_process/simple_mp/build/simple_mp mapped=44 N1=44 kernelpagesize_kB=64
00ad0000 default file=/data1/dpdk-19.11/examples/multi_process/simple_mp/build/simple_mp anon=1 dirty=1 mapped=2 N0=1 N1=1 kernelpagesize_kB=64
00af0000 default file=/data1/dpdk-19.11/examples/multi_process/simple_mp/build/simple_mp anon=4 dirty=4 N3=4 kernelpagesize_kB=64
00b30000 default anon=21 dirty=21 N0=15 N3=6 kernelpagesize_kB=64
0ead0000 default heap anon=42 dirty=42 N0=15 N3=27 kernelpagesize_kB=64
100000000 default file=/run/dpdk/rte/config dirty=1 N0=1 kernelpagesize_kB=64
100010000 default file=/run/dpdk/rte/fbarray_memzone dirty=3 N0=3 kernelpagesize_kB=64
100040000 default file=/run/dpdk/rte/fbarray_memseg-524288k-0-0 dirty=1 N0=1 kernelpagesize_kB=64
120000000 default
920000000 default file=/run/dpdk/rte/fbarray_memseg-524288k-0-1 dirty=1 N0=1 kernelpagesize_kB=64
940000000 default
1140000000 default file=/run/dpdk/rte/fbarray_memseg-524288k-1-0 dirty=1 N0=1 kernelpagesize_kB=64
1160000000 default
1960000000 default file=/run/dpdk/rte/fbarray_memseg-524288k-1-1 dirty=1 N0=1 kernelpagesize_kB=64
1980000000 default
2180000000 default file=/run/dpdk/rte/fbarray_memseg-524288k-2-0 dirty=1 N0=1 kernelpagesize_kB=64
21a0000000 default
29a0000000 default file=/run/dpdk/rte/fbarray_memseg-524288k-2-1 dirty=1 N0=1 kernelpagesize_kB=64
29c0000000 default
31c0000000 default file=/run/dpdk/rte/fbarray_memseg-524288k-3-0 dirty=1 N0=1 kernelpagesize_kB=64
31e0000000 default
39e0000000 default file=/run/dpdk/rte/fbarray_memseg-524288k-3-1 dirty=1 N0=1 kernelpagesize_kB=64
3a00000000 default
4200000000 default file=/run/dpdk/rte/fbarray_memseg-2048k-0-0 dirty=7 N0=7 kernelpagesize_kB=64
4200200000 default
4600200000 default file=/run/dpdk/rte/fbarray_memseg-2048k-0-1 dirty=7 N0=7 kernelpagesize_kB=64
4600400000 default
4a00400000 default file=/run/dpdk/rte/fbarray_memseg-2048k-0-2 dirty=7 N0=7 kernelpagesize_kB=64
4a00600000 default
4e00600000 default file=/run/dpdk/rte/fbarray_memseg-2048k-0-3 dirty=7 N0=7 kernelpagesize_kB=64
4e00800000 default
5200800000 default file=/run/dpdk/rte/fbarray_memseg-2048k-1-0 dirty=7 N0=7 kernelpagesize_kB=64
5200a00000 default
5600a00000 default file=/run/dpdk/rte/fbarray_memseg-2048k-1-1 dirty=7 N0=7 kernelpagesize_kB=64
5600c00000 default
5a00c00000 default file=/run/dpdk/rte/fbarray_memseg-2048k-1-2 dirty=7 N0=7 kernelpagesize_kB=64
5a00e00000 default
5e00e00000 default file=/run/dpdk/rte/fbarray_memseg-2048k-1-3 dirty=7 N0=7 kernelpagesize_kB=64
5e01000000 default
6201000000 default file=/run/dpdk/rte/fbarray_memseg-2048k-2-0 dirty=7 N0=7 kernelpagesize_kB=64
6201200000 default
6601200000 default file=/run/dpdk/rte/fbarray_memseg-2048k-2-1 dirty=7 N0=7 kernelpagesize_kB=64
6601400000 default
6a01400000 default file=/run/dpdk/rte/fbarray_memseg-2048k-2-2 dirty=7 N0=7 kernelpagesize_kB=64
6a01600000 default
6e01600000 default file=/run/dpdk/rte/fbarray_memseg-2048k-2-3 dirty=7 N0=7 kernelpagesize_kB=64
6e01800000 default
7201800000 default file=/run/dpdk/rte/fbarray_memseg-2048k-3-0 dirty=7 N0=7 kernelpagesize_kB=64
7201a00000 default file=/mnt/huge/rtemap_163840 huge dirty=1 N3=1 kernelpagesize_kB=2048
7201c00000 default file=/mnt/huge/rtemap_163841 huge dirty=1 N3=1 kernelpagesize_kB=2048
7201e00000 default file=/mnt/huge/rtemap_163842 huge dirty=1 N3=1 kernelpagesize_kB=2048
7202000000 default
7601a00000 default file=/run/dpdk/rte/fbarray_memseg-2048k-3-1 dirty=7 N0=7 kernelpagesize_kB=64
7601c00000 default
7a01c00000 default file=/run/dpdk/rte/fbarray_memseg-2048k-3-2 dirty=7 N0=7 kernelpagesize_kB=64
7a01e00000 default
7e01e00000 default file=/run/dpdk/rte/fbarray_memseg-2048k-3-3 dirty=7 N0=7 kernelpagesize_kB=64
7e02000000 default
ffffb8cc0000 default
ffffb8cd0000 default anon=2 dirty=2 N0=1 N3=1 kernelpagesize_kB=64
ffffb94e0000 default
ffffb94f0000 default anon=1 dirty=1 N0=1 kernelpagesize_kB=64
ffffb9cf0000 default
ffffb9d00000 default anon=1 dirty=1 N0=1 kernelpagesize_kB=64
ffffba500000 default file=/usr/lib64/libgcc_s-4.8.5-20150702.so.1 mapped=1 mapmax=20 N2=1 kernelpagesize_kB=64
ffffba520000 default file=/usr/lib64/libgcc_s-4.8.5-20150702.so.1 anon=1 dirty=1 N0=1 kernelpagesize_kB=64
ffffba530000 default file=/usr/lib64/libgcc_s-4.8.5-20150702.so.1 anon=1 dirty=1 N0=1 kernelpagesize_kB=64
ffffba540000 default file=/usr/lib64/libc-2.17.so mapped=19 mapmax=31 N2=19 kernelpagesize_kB=64
ffffba6b0000 default file=/usr/lib64/libc-2.17.so anon=1 dirty=1 N0=1 kernelpagesize_kB=64
ffffba6c0000 default file=/usr/lib64/libc-2.17.so anon=1 dirty=1 N0=1 kernelpagesize_kB=64
ffffba6d0000 default file=/usr/lib64/libpthread-2.17.so mapped=2 mapmax=21 N2=2 kernelpagesize_kB=64
ffffba6f0000 default file=/usr/lib64/libpthread-2.17.so anon=1 dirty=1 N0=1 kernelpagesize_kB=64
ffffba700000 default file=/usr/lib64/libpthread-2.17.so anon=1 dirty=1 N0=1 kernelpagesize_kB=64
ffffba710000 default file=/usr/lib64/libdl-2.17.so mapped=1 mapmax=27 N2=1 kernelpagesize_kB=64
ffffba720000 default file=/usr/lib64/libdl-2.17.so anon=1 dirty=1 N0=1 kernelpagesize_kB=64
ffffba730000 default file=/usr/lib64/libdl-2.17.so anon=1 dirty=1 N0=1 kernelpagesize_kB=64
ffffba740000 default file=/usr/lib64/libnuma.so.1.0.0 mapped=1 mapmax=2 N0=1 kernelpagesize_kB=64
ffffba750000 default file=/usr/lib64/libnuma.so.1.0.0 anon=1 dirty=1 N0=1 kernelpagesize_kB=64
ffffba760000 default file=/usr/lib64/libnuma.so.1.0.0 anon=1 dirty=1 N0=1 kernelpagesize_kB=64
ffffba770000 default file=/usr/lib64/libm-2.17.so mapped=1 mapmax=17 N0=1 kernelpagesize_kB=64
ffffba810000 default file=/usr/lib64/libm-2.17.so anon=1 dirty=1 N0=1 kernelpagesize_kB=64
ffffba820000 default file=/usr/lib64/libm-2.17.so anon=1 dirty=1 N0=1 kernelpagesize_kB=64
ffffba830000 default file=/usr/lib64/librt-2.17.so mapped=1 mapmax=20 N2=1 kernelpagesize_kB=64
ffffba840000 default file=/usr/lib64/librt-2.17.so anon=1 dirty=1 N0=1 kernelpagesize_kB=64
ffffba850000 default file=/usr/lib64/librt-2.17.so anon=1 dirty=1 N0=1 kernelpagesize_kB=64
ffffba860000 default anon=1 dirty=1 N0=1 kernelpagesize_kB=64
ffffba870000 default
ffffba880000 default
ffffba890000 default file=/usr/lib64/ld-2.17.so mapped=2 mapmax=31 N2=2 kernelpagesize_kB=64
ffffba8b0000 default file=/usr/lib64/ld-2.17.so anon=1 dirty=1 N0=1 kernelpagesize_kB=64
ffffba8c0000 default file=/usr/lib64/ld-2.17.so anon=1 dirty=1 N0=1 kernelpagesize_kB=64
fffffa700000 default stack anon=1 dirty=1 N0=1 kernelpagesize_kB=64
[root@localhost simple_mp]

[root@localhost simple_mp]# cat /sys/devices/system/node/node0/meminfo
Node 0 MemTotal:       133176640 kB
Node 0 MemFree:        96906880 kB
Node 0 MemUsed:        36269760 kB
Node 0 Active:          1257024 kB
Node 0 Inactive:         387328 kB
Node 0 Active(anon):      73152 kB
Node 0 Inactive(anon):      448 kB
Node 0 Active(file):    1183872 kB
Node 0 Inactive(file):   386880 kB
Node 0 Unevictable:           0 kB
Node 0 Mlocked:               0 kB
Node 0 Dirty:                 0 kB
Node 0 Writeback:             0 kB
Node 0 FilePages:       1580160 kB
Node 0 Mapped:            11712 kB
Node 0 AnonPages:         65536 kB
Node 0 Shmem:              9408 kB
Node 0 KernelStack:       27328 kB
Node 0 PageTables:         4032 kB
Node 0 NFS_Unstable:          0 kB
Node 0 Bounce:                0 kB
Node 0 WritebackTmp:          0 kB
Node 0 Slab:             367936 kB
Node 0 SReclaimable:     110208 kB
Node 0 SUnreclaim:       257728 kB
Node 0 AnonHugePages:         0 kB
Node 0 ShmemHugePages:        0 kB
Node 0 ShmemPmdMapped:        0 kB
Node 0 HugePages_Total:    64
Node 0 HugePa