KSM Demo 分析

KSM是“Kernel SamePage Merging ”的缩写，中文可称为“内核同页合并 ”。它是一种节省内存的技术，从2.6.32版开始获得支持，内核需要打开CONFIG_KSM=y选项使用KSM机制。

KSM允许内核在两个或多个进程（包括虚拟客户机）之间共享完全相同的内存页。KSM让内核扫描检查 正在运行中的程序 并比较它们的内存 ，如果发现它们有完全相同的内存区域或内存页 ，就将多个相同的内存合并为一个单一的内存页，并将其标识为“写时复制 ”。这样可以起到节省系统内存使用量的作用。之后，如果有进程试图去修改被标识为“写时复制”的合并内存页，就为该进程复制出一个新的内存页供其使用。

KSM最初是为与KVM一起使用而开发的(在那里它被称为内核共享内存)，通过共享虚拟机之间的公共数据来支持更多的虚拟机同时运行。但是这种设计对那些有大量相同数据的应用程序同样适用。

KSM只合并匿名(私有)页面，从不合并pagecache(文件)页面。可以参考文档获取更多信息：

Documentation/admin-guide/mm/ksm.rst

用户需要调用madvise显示声明需要支持KSM的区域，传入参数MADV_MERGEABLE。

int madvise(addr, length, MADV_MERGEABLE)

并且还可以通过传入MADV_UNMERGEABLE撤销此区域对KSM的支持。

int madvise(addr, length, MADV_UNMERGEABLE)

ksm.c

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>

int main(int argc, char **argv)
{
	char *buf;
	char filename[64] = "";
	struct stat stat;

	int size = 100 * 4096;
	int fd = 0;

	strcpy(filename, argv[1]);

	fd = open(filename, O_RDWR | O_CREAT, 0664);
	if (fd < 0) {
		printf("%s line %d, open file failure.\n",
		       __func__, __LINE__);
		return -1;
	}

	fstat(fd, &stat);

	buf = mmap(NULL, stat.st_size, PROT_WRITE, MAP_PRIVATE, fd, 0);
	if (buf == MAP_FAILED) {
		printf("%s line %d, map failure.\n",
		       __func__, __LINE__);
		return -1;
	}

	memset(buf, 0x5a, stat.st_size);

	madvise(buf, stat.st_size, MADV_MERGEABLE);
	while (1)
		sleep(1);

	return 0;
}

创建mmap文件

dd if=/dev/zero of=./ksm.dat bs=1M count=100

运行：

./a.out ksm.dat &

可以看到ksm.dat文件映射了100M的虚拟地址空间。

开启KSM

# echo 1 > /sys/kernel/mm/ksm/run

稍等片刻 查看如下几个文件节点 

/sys/kernel/mm/ksm/pages_shared
/sys/kernel/mm/ksm/pages_sharing

100M的空间，每页4K，一共有25600个页，经过KSM 页面共享之后，有25500个页面共享了100个页面，实际只消耗了4个PAGE。

我们用工具逆向查看这100M的物理地址分布。

查看100个PFN的分布：

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <assert.h>
#include <errno.h>
#include <stdint.h>
#include <string.h>

#define PAGEMAP_ENTRY     8
#define GET_BIT(X,Y)      (X & ((uint64_t)1<<Y)) >> Y
#define GET_PFN(X)        X & 0x7FFFFFFFFFFFFF

const int __endian_bit = 1;
#define is_bigendian()   ((*(char*)&__endian_bit) == 0)

static int read_pagemap(char *path_buf, unsigned long virt_addr, int page_size, int len);

int main(int argc, char **argv)
{
	int page_size;
	static char path_buf[0x100] = {  };
	char *end;
	int pid;
	unsigned long virt_addr;
	int len;

	if (argc != 4) {
		printf("Argument number is not correct!\n pagemap PID VIRTUAL_ADDRESS\n");
		return -1;
	}

	if (!memcmp(argv[1], "self", sizeof("self"))) {
		sprintf(path_buf, "/proc/self/pagemap");
		pid = -1;
	} else {
		pid = strtol(argv[1], &end, 10);
		if (end == argv[1] || *end != '\0' || pid <= 0) {
			printf("PID must be a positive number or 'self'\n");
			return -1;
		}
	}

	virt_addr = strtoll(argv[2], NULL, 16);
	if (pid != -1) {
		sprintf(path_buf, "/proc/%u/pagemap", pid);
	}

	len = atoi(argv[3]);

	page_size = getpagesize();
	read_pagemap(path_buf, virt_addr, page_size, len*1024*1024);

	return 0;
}

int read_pagemap(char *path_buf, unsigned long virt_addr, int page_size, int len)
{
	FILE *f;
	uint64_t file_offset;
	unsigned char c_buf[PAGEMAP_ENTRY];
	uint64_t read_val = 0;
	int status, i, c;
	int j;
	long old_pfn = -1, idx = 0;
	unsigned long tmp_vaddr;

	f = fopen(path_buf, "rb");
	if (!f) {
		printf("error! cannot open %s.\n", path_buf);
		return -1;
	}

	for(j = 0; j < len / page_size; j ++) {
		//Shifting by virt-addr-offset number of bytes
		//and multiplying by the size of an address (the size of an entry in pagemap file)
		
		tmp_vaddr = virt_addr + j * page_size;
		file_offset = tmp_vaddr / page_size * PAGEMAP_ENTRY;

		//printf("Vaddr: 0x%lx, Page_size: %d, Entry_size: %d\n", tmp_vaddr, page_size, PAGEMAP_ENTRY);
		//printf("Reading %s at 0x%llx\n", path_buf, (unsigned long long) file_offset);

		status = fseek(f, file_offset, SEEK_SET);
		if (status) {
			perror("failed to do fseek!");
			return -1;
		}

		for (i = 0; i < PAGEMAP_ENTRY; i++) {
			c = getc(f);
			if (c == EOF) {
				printf("reached end of the file\n");
				return 0;
			}
			if (is_bigendian())
				c_buf[i] = c;
			else
				c_buf[PAGEMAP_ENTRY - i - 1] = c;

			//printf("[%d]0x%x ", i, c);
		}

		for (i = 0; i < PAGEMAP_ENTRY; i++) {
			//printf("%d ",c_buf[i]);
			read_val = (read_val << 8) + c_buf[i];
		}

		//printf("\n");
		//printf("result: 0x%llx\n", (unsigned long long) read_val);

		if (GET_BIT(read_val, 63)) {
			uint64_t pfn = GET_PFN(read_val);
			//printf("PFN: 0x%lx (0x%lx)\n", pfn, pfn * page_size + tmp_vaddr % page_size);
			if(old_pfn == -1) {
				printf("[%ld]: 0x%lx, pfn 0x%lx.\n", idx, tmp_vaddr, pfn);
				idx ++;
			} else if(old_pfn != pfn){
				printf("[%ld]: 0x%lx, pfn 0x%lx.\n", idx, tmp_vaddr, pfn);
				idx ++;
			} else {
			}
			old_pfn = pfn;
		} else {
			printf("Page not present, 0x%lx\n", tmp_vaddr);
		}

		if (GET_BIT(read_val, 62)) {
			printf("Page swapped\n");
		}
	}

	fclose(f);

	return 0;
}

反查页表得到物理地址

zlcao@zlcao-Vostro-3268:~/Workspace/linux/dumpstack/test$ sudo ./a.out 18067 0x7f6931ae7000 100
[0]: 0x7f6931ae7000, pfn 0x1c763b.
[1]: 0x7f6931be7000, pfn 0x10c5a4.
[2]: 0x7f6931ce7000, pfn 0x12f8d5.
[3]: 0x7f6931de7000, pfn 0x1f7ca5.
[4]: 0x7f6931ee7000, pfn 0x1b3ad8.
[5]: 0x7f6931fe7000, pfn 0x25d961.
[6]: 0x7f69320e7000, pfn 0x11795c.
[7]: 0x7f69321e7000, pfn 0x21a9d7.
[8]: 0x7f69322e7000, pfn 0x229a21.
[9]: 0x7f69323e7000, pfn 0x237f51.
[10]: 0x7f69324e7000, pfn 0x1a77fb.
[11]: 0x7f69325e7000, pfn 0x187731.
[12]: 0x7f69326e7000, pfn 0x182889.
[13]: 0x7f69327e7000, pfn 0x1b812b.
[14]: 0x7f69328e7000, pfn 0x124609.
[15]: 0x7f69329e7000, pfn 0x19a7a3.
[16]: 0x7f6932ae7000, pfn 0x15e9b9.
[17]: 0x7f6932be7000, pfn 0x1eab61.
[18]: 0x7f6932ce7000, pfn 0x25dc39.
[19]: 0x7f6932de7000, pfn 0x1414cf.
[20]: 0x7f6932ee7000, pfn 0x10bf0b.
[21]: 0x7f6932fe7000, pfn 0x24074f.
[22]: 0x7f69330e7000, pfn 0x19923b.
[23]: 0x7f69331e7000, pfn 0x1a1853.
[24]: 0x7f69332e7000, pfn 0x1a182f.
[25]: 0x7f69333e7000, pfn 0x101637.
[26]: 0x7f69334e7000, pfn 0x20d179.
[27]: 0x7f69335e7000, pfn 0x157713.
[28]: 0x7f69336e7000, pfn 0x1ec6a7.
[29]: 0x7f69337e7000, pfn 0x121445.
[30]: 0x7f69338e7000, pfn 0x1ef77f.
[31]: 0x7f69339e7000, pfn 0x22005f.
[32]: 0x7f6933ae7000, pfn 0x255b7f.
[33]: 0x7f6933be7000, pfn 0x1211f7.
[34]: 0x7f6933ce7000, pfn 0x1d19bf.
[35]: 0x7f6933de7000, pfn 0x1ff3f5.
[36]: 0x7f6933ee7000, pfn 0x235321.
[37]: 0x7f6933fe7000, pfn 0x10fe2d.
[38]: 0x7f69340e7000, pfn 0x11d24b.
[39]: 0x7f69341e7000, pfn 0x220459.
[40]: 0x7f69342e7000, pfn 0x15ac09.
[41]: 0x7f69343e7000, pfn 0x109f7d.
[42]: 0x7f69344e7000, pfn 0x1983bf.
[43]: 0x7f69345e7000, pfn 0x1b94ff.
[44]: 0x7f69346e7000, pfn 0x1b9d5f.
[45]: 0x7f69347e7000, pfn 0x2325eb.
[46]: 0x7f69348e7000, pfn 0x1d83cf.
[47]: 0x7f69349e7000, pfn 0x18adaf.
[48]: 0x7f6934ae7000, pfn 0x1de197.
[49]: 0x7f6934be7000, pfn 0x2485ab.
[50]: 0x7f6934ce7000, pfn 0x115b6b.
[51]: 0x7f6934de7000, pfn 0x102cb3.
[52]: 0x7f6934ee7000, pfn 0x1731cb.
[53]: 0x7f6934fe7000, pfn 0x24cf9b.
[54]: 0x7f69350e7000, pfn 0x198b17.
[55]: 0x7f69351e7000, pfn 0x1a60e7.
[56]: 0x7f69352e7000, pfn 0x1a281b.
[57]: 0x7f69353e7000, pfn 0x1b41df.
[58]: 0x7f69354e7000, pfn 0x1cbdf3.
[59]: 0x7f69355e7000, pfn 0x210e3f.
[60]: 0x7f69356e7000, pfn 0x13e2c7.
[61]: 0x7f69357e7000, pfn 0x21618f.
[62]: 0x7f69358e7000, pfn 0x1ef21b.
[63]: 0x7f69359e7000, pfn 0x20c5b3.
[64]: 0x7f6935ae7000, pfn 0x19f1a3.
[65]: 0x7f6935be7000, pfn 0x10b93b.
[66]: 0x7f6935ce7000, pfn 0x153813.
[67]: 0x7f6935de7000, pfn 0x22f13b.
[68]: 0x7f6935ee7000, pfn 0x171a2b.
[69]: 0x7f6935fe7000, pfn 0x17bb53.
[70]: 0x7f69360e7000, pfn 0x1b0a9b.
[71]: 0x7f69361e7000, pfn 0x1ee77b.
[72]: 0x7f69362e7000, pfn 0x2334a3.
[73]: 0x7f69363e7000, pfn 0x231bb3.
[74]: 0x7f69364e7000, pfn 0x2065bb.
[75]: 0x7f69365e7000, pfn 0x12f53b.
[76]: 0x7f69366e7000, pfn 0x201223.
[77]: 0x7f69367e7000, pfn 0x1555a3.
[78]: 0x7f69368e7000, pfn 0x19f8c3.
[79]: 0x7f69369e7000, pfn 0x1a97ab.
[80]: 0x7f6936ae7000, pfn 0x181fbb.
[81]: 0x7f6936be7000, pfn 0x17d83b.
[82]: 0x7f6936ce7000, pfn 0x153cab.
[83]: 0x7f6936de7000, pfn 0x225b63.
[84]: 0x7f6936ee7000, pfn 0x211a9b.
[85]: 0x7f6936fe7000, pfn 0x19882b.
[86]: 0x7f69370e7000, pfn 0x14d79b.
[87]: 0x7f69371e7000, pfn 0x1fc5bb.
[88]: 0x7f69372e7000, pfn 0x1a17eb.
[89]: 0x7f69373e7000, pfn 0x1bb6cb.
[90]: 0x7f69374e7000, pfn 0x241ebb.
[91]: 0x7f69375e7000, pfn 0x23c08b.
[92]: 0x7f69376e7000, pfn 0x24b04b.
[93]: 0x7f69377e7000, pfn 0x1ee09b.
[94]: 0x7f69378e7000, pfn 0x11c9bb.
[95]: 0x7f69379e7000, pfn 0x14763b.
[96]: 0x7f6937ae7000, pfn 0x18473b.
[97]: 0x7f6937be7000, pfn 0x18d0eb.
[98]: 0x7f6937ce7000, pfn 0x1db76b.
[99]: 0x7f6937de7000, pfn 0x160c2b.

可以看到，100M的内存区间，从0x7f6931ae7000地址开始，每1M映射一个新的KSM页面，1M内则共享同一个物理页面，所以，100M的空间实际上只映射了100个物理页面，上面的例子中我们找到了所有的100个物理页面。

KSM的应用

为了在有限的内存中运行更多的虚拟机，KSM在虚拟化技术中应用广泛，如果把物理机看成小区，虚拟机看成是房间，则KSM触发的页面合并则是这个小区的公谈面积。

具体可以结合QEMU或者KVMTOOL的代码分析。

KVM 介绍（2）：CPU 和内存虚拟化_kvm socket cores-CSDN博客

---

结束