KSM是“Kernel SamePage Merging ”的缩写,中文可称为“内核同页合并 ”。它是一种节省内存的技术,从2.6.32版开始获得支持,内核需要打开CONFIG_KSM=y选项使用KSM机制。
KSM允许内核在两个或多个进程(包括虚拟客户机)之间共享完全相同的内存页。KSM让内核扫描检查 正在运行中的程序 并比较它们的内存 ,如果发现它们有完全相同的内存区域或内存页 ,就将多个相同的内存合并为一个单一的内存页,并将其标识为“写时复制 ”。这样可以起到节省系统内存使用量的作用。之后,如果有进程试图去修改被标识为“写时复制”的合并内存页,就为该进程复制出一个新的内存页供其使用。
KSM最初是为与KVM一起使用而开发的(在那里它被称为内核共享内存),通过共享虚拟机之间的公共数据来支持更多的虚拟机同时运行。但是这种设计对那些有大量相同数据的应用程序同样适用。
KSM只合并匿名(私有)页面,从不合并pagecache(文件)页面。可以参考文档获取更多信息:
Documentation/admin-guide/mm/ksm.rst
用户需要调用madvise显示声明需要支持KSM的区域,传入参数MADV_MERGEABLE。
int madvise(addr, length, MADV_MERGEABLE)
并且还可以通过传入MADV_UNMERGEABLE撤销此区域对KSM的支持。
int madvise(addr, length, MADV_UNMERGEABLE)
ksm.c
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
int main(int argc, char **argv)
{
char *buf;
char filename[64] = "";
struct stat stat;
int size = 100 * 4096;
int fd = 0;
strcpy(filename, argv[1]);
fd = open(filename, O_RDWR | O_CREAT, 0664);
if (fd < 0) {
printf("%s line %d, open file failure.\n",
__func__, __LINE__);
return -1;
}
fstat(fd, &stat);
buf = mmap(NULL, stat.st_size, PROT_WRITE, MAP_PRIVATE, fd, 0);
if (buf == MAP_FAILED) {
printf("%s line %d, map failure.\n",
__func__, __LINE__);
return -1;
}
memset(buf, 0x5a, stat.st_size);
madvise(buf, stat.st_size, MADV_MERGEABLE);
while (1)
sleep(1);
return 0;
}
创建mmap文件
dd if=/dev/zero of=./ksm.dat bs=1M count=100
运行:
./a.out ksm.dat &
可以看到ksm.dat文件映射了100M的虚拟地址空间。
开启KSM
# echo 1 > /sys/kernel/mm/ksm/run
稍等片刻 查看如下几个文件节点
/sys/kernel/mm/ksm/pages_shared
/sys/kernel/mm/ksm/pages_sharing
100M的空间,每页4K,一共有25600个页,经过KSM 页面共享之后,有25500个页面共享了100个页面,实际只消耗了4个PAGE。
我们用工具逆向查看这100M的物理地址分布。
查看100个PFN的分布:
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <assert.h>
#include <errno.h>
#include <stdint.h>
#include <string.h>
#define PAGEMAP_ENTRY 8
#define GET_BIT(X,Y) (X & ((uint64_t)1<<Y)) >> Y
#define GET_PFN(X) X & 0x7FFFFFFFFFFFFF
const int __endian_bit = 1;
#define is_bigendian() ((*(char*)&__endian_bit) == 0)
static int read_pagemap(char *path_buf, unsigned long virt_addr, int page_size, int len);
int main(int argc, char **argv)
{
int page_size;
static char path_buf[0x100] = { };
char *end;
int pid;
unsigned long virt_addr;
int len;
if (argc != 4) {
printf("Argument number is not correct!\n pagemap PID VIRTUAL_ADDRESS\n");
return -1;
}
if (!memcmp(argv[1], "self", sizeof("self"))) {
sprintf(path_buf, "/proc/self/pagemap");
pid = -1;
} else {
pid = strtol(argv[1], &end, 10);
if (end == argv[1] || *end != '\0' || pid <= 0) {
printf("PID must be a positive number or 'self'\n");
return -1;
}
}
virt_addr = strtoll(argv[2], NULL, 16);
if (pid != -1) {
sprintf(path_buf, "/proc/%u/pagemap", pid);
}
len = atoi(argv[3]);
page_size = getpagesize();
read_pagemap(path_buf, virt_addr, page_size, len*1024*1024);
return 0;
}
int read_pagemap(char *path_buf, unsigned long virt_addr, int page_size, int len)
{
FILE *f;
uint64_t file_offset;
unsigned char c_buf[PAGEMAP_ENTRY];
uint64_t read_val = 0;
int status, i, c;
int j;
long old_pfn = -1, idx = 0;
unsigned long tmp_vaddr;
f = fopen(path_buf, "rb");
if (!f) {
printf("error! cannot open %s.\n", path_buf);
return -1;
}
for(j = 0; j < len / page_size; j ++) {
//Shifting by virt-addr-offset number of bytes
//and multiplying by the size of an address (the size of an entry in pagemap file)
tmp_vaddr = virt_addr + j * page_size;
file_offset = tmp_vaddr / page_size * PAGEMAP_ENTRY;
//printf("Vaddr: 0x%lx, Page_size: %d, Entry_size: %d\n", tmp_vaddr, page_size, PAGEMAP_ENTRY);
//printf("Reading %s at 0x%llx\n", path_buf, (unsigned long long) file_offset);
status = fseek(f, file_offset, SEEK_SET);
if (status) {
perror("failed to do fseek!");
return -1;
}
for (i = 0; i < PAGEMAP_ENTRY; i++) {
c = getc(f);
if (c == EOF) {
printf("reached end of the file\n");
return 0;
}
if (is_bigendian())
c_buf[i] = c;
else
c_buf[PAGEMAP_ENTRY - i - 1] = c;
//printf("[%d]0x%x ", i, c);
}
for (i = 0; i < PAGEMAP_ENTRY; i++) {
//printf("%d ",c_buf[i]);
read_val = (read_val << 8) + c_buf[i];
}
//printf("\n");
//printf("result: 0x%llx\n", (unsigned long long) read_val);
if (GET_BIT(read_val, 63)) {
uint64_t pfn = GET_PFN(read_val);
//printf("PFN: 0x%lx (0x%lx)\n", pfn, pfn * page_size + tmp_vaddr % page_size);
if(old_pfn == -1) {
printf("[%ld]: 0x%lx, pfn 0x%lx.\n", idx, tmp_vaddr, pfn);
idx ++;
} else if(old_pfn != pfn){
printf("[%ld]: 0x%lx, pfn 0x%lx.\n", idx, tmp_vaddr, pfn);
idx ++;
} else {
}
old_pfn = pfn;
} else {
printf("Page not present, 0x%lx\n", tmp_vaddr);
}
if (GET_BIT(read_val, 62)) {
printf("Page swapped\n");
}
}
fclose(f);
return 0;
}
反查页表得到物理地址
zlcao@zlcao-Vostro-3268:~/Workspace/linux/dumpstack/test$ sudo ./a.out 18067 0x7f6931ae7000 100
[0]: 0x7f6931ae7000, pfn 0x1c763b.
[1]: 0x7f6931be7000, pfn 0x10c5a4.
[2]: 0x7f6931ce7000, pfn 0x12f8d5.
[3]: 0x7f6931de7000, pfn 0x1f7ca5.
[4]: 0x7f6931ee7000, pfn 0x1b3ad8.
[5]: 0x7f6931fe7000, pfn 0x25d961.
[6]: 0x7f69320e7000, pfn 0x11795c.
[7]: 0x7f69321e7000, pfn 0x21a9d7.
[8]: 0x7f69322e7000, pfn 0x229a21.
[9]: 0x7f69323e7000, pfn 0x237f51.
[10]: 0x7f69324e7000, pfn 0x1a77fb.
[11]: 0x7f69325e7000, pfn 0x187731.
[12]: 0x7f69326e7000, pfn 0x182889.
[13]: 0x7f69327e7000, pfn 0x1b812b.
[14]: 0x7f69328e7000, pfn 0x124609.
[15]: 0x7f69329e7000, pfn 0x19a7a3.
[16]: 0x7f6932ae7000, pfn 0x15e9b9.
[17]: 0x7f6932be7000, pfn 0x1eab61.
[18]: 0x7f6932ce7000, pfn 0x25dc39.
[19]: 0x7f6932de7000, pfn 0x1414cf.
[20]: 0x7f6932ee7000, pfn 0x10bf0b.
[21]: 0x7f6932fe7000, pfn 0x24074f.
[22]: 0x7f69330e7000, pfn 0x19923b.
[23]: 0x7f69331e7000, pfn 0x1a1853.
[24]: 0x7f69332e7000, pfn 0x1a182f.
[25]: 0x7f69333e7000, pfn 0x101637.
[26]: 0x7f69334e7000, pfn 0x20d179.
[27]: 0x7f69335e7000, pfn 0x157713.
[28]: 0x7f69336e7000, pfn 0x1ec6a7.
[29]: 0x7f69337e7000, pfn 0x121445.
[30]: 0x7f69338e7000, pfn 0x1ef77f.
[31]: 0x7f69339e7000, pfn 0x22005f.
[32]: 0x7f6933ae7000, pfn 0x255b7f.
[33]: 0x7f6933be7000, pfn 0x1211f7.
[34]: 0x7f6933ce7000, pfn 0x1d19bf.
[35]: 0x7f6933de7000, pfn 0x1ff3f5.
[36]: 0x7f6933ee7000, pfn 0x235321.
[37]: 0x7f6933fe7000, pfn 0x10fe2d.
[38]: 0x7f69340e7000, pfn 0x11d24b.
[39]: 0x7f69341e7000, pfn 0x220459.
[40]: 0x7f69342e7000, pfn 0x15ac09.
[41]: 0x7f69343e7000, pfn 0x109f7d.
[42]: 0x7f69344e7000, pfn 0x1983bf.
[43]: 0x7f69345e7000, pfn 0x1b94ff.
[44]: 0x7f69346e7000, pfn 0x1b9d5f.
[45]: 0x7f69347e7000, pfn 0x2325eb.
[46]: 0x7f69348e7000, pfn 0x1d83cf.
[47]: 0x7f69349e7000, pfn 0x18adaf.
[48]: 0x7f6934ae7000, pfn 0x1de197.
[49]: 0x7f6934be7000, pfn 0x2485ab.
[50]: 0x7f6934ce7000, pfn 0x115b6b.
[51]: 0x7f6934de7000, pfn 0x102cb3.
[52]: 0x7f6934ee7000, pfn 0x1731cb.
[53]: 0x7f6934fe7000, pfn 0x24cf9b.
[54]: 0x7f69350e7000, pfn 0x198b17.
[55]: 0x7f69351e7000, pfn 0x1a60e7.
[56]: 0x7f69352e7000, pfn 0x1a281b.
[57]: 0x7f69353e7000, pfn 0x1b41df.
[58]: 0x7f69354e7000, pfn 0x1cbdf3.
[59]: 0x7f69355e7000, pfn 0x210e3f.
[60]: 0x7f69356e7000, pfn 0x13e2c7.
[61]: 0x7f69357e7000, pfn 0x21618f.
[62]: 0x7f69358e7000, pfn 0x1ef21b.
[63]: 0x7f69359e7000, pfn 0x20c5b3.
[64]: 0x7f6935ae7000, pfn 0x19f1a3.
[65]: 0x7f6935be7000, pfn 0x10b93b.
[66]: 0x7f6935ce7000, pfn 0x153813.
[67]: 0x7f6935de7000, pfn 0x22f13b.
[68]: 0x7f6935ee7000, pfn 0x171a2b.
[69]: 0x7f6935fe7000, pfn 0x17bb53.
[70]: 0x7f69360e7000, pfn 0x1b0a9b.
[71]: 0x7f69361e7000, pfn 0x1ee77b.
[72]: 0x7f69362e7000, pfn 0x2334a3.
[73]: 0x7f69363e7000, pfn 0x231bb3.
[74]: 0x7f69364e7000, pfn 0x2065bb.
[75]: 0x7f69365e7000, pfn 0x12f53b.
[76]: 0x7f69366e7000, pfn 0x201223.
[77]: 0x7f69367e7000, pfn 0x1555a3.
[78]: 0x7f69368e7000, pfn 0x19f8c3.
[79]: 0x7f69369e7000, pfn 0x1a97ab.
[80]: 0x7f6936ae7000, pfn 0x181fbb.
[81]: 0x7f6936be7000, pfn 0x17d83b.
[82]: 0x7f6936ce7000, pfn 0x153cab.
[83]: 0x7f6936de7000, pfn 0x225b63.
[84]: 0x7f6936ee7000, pfn 0x211a9b.
[85]: 0x7f6936fe7000, pfn 0x19882b.
[86]: 0x7f69370e7000, pfn 0x14d79b.
[87]: 0x7f69371e7000, pfn 0x1fc5bb.
[88]: 0x7f69372e7000, pfn 0x1a17eb.
[89]: 0x7f69373e7000, pfn 0x1bb6cb.
[90]: 0x7f69374e7000, pfn 0x241ebb.
[91]: 0x7f69375e7000, pfn 0x23c08b.
[92]: 0x7f69376e7000, pfn 0x24b04b.
[93]: 0x7f69377e7000, pfn 0x1ee09b.
[94]: 0x7f69378e7000, pfn 0x11c9bb.
[95]: 0x7f69379e7000, pfn 0x14763b.
[96]: 0x7f6937ae7000, pfn 0x18473b.
[97]: 0x7f6937be7000, pfn 0x18d0eb.
[98]: 0x7f6937ce7000, pfn 0x1db76b.
[99]: 0x7f6937de7000, pfn 0x160c2b.
可以看到,100M的内存区间,从0x7f6931ae7000地址开始,每1M映射一个新的KSM页面,1M内则共享同一个物理页面,所以,100M的空间实际上只映射了100个物理页面,上面的例子中我们找到了所有的100个物理页面。
KSM的应用
为了在有限的内存中运行更多的虚拟机,KSM在虚拟化技术中应用广泛,如果把物理机看成小区,虚拟机看成是房间,则KSM触发的页面合并则是这个小区的公谈面积。
具体可以结合QEMU或者KVMTOOL的代码分析。
KVM 介绍(2):CPU 和内存虚拟化_kvm socket cores-CSDN博客