per-cpu 变量

概述

per-cpu 变量是 2.6 内核的一个有趣特性。当建立一个 per-cpu 变量时，系统中的每个处理器都会拥有该变量的特有副本。这看起来很奇怪，但它有其优点。对 per-cpu 变量的访问（几乎）不需要锁定，因为每个处理器在其自己的副本上工作。per-cpu 变量还可以保存在对应处理器的高速缓存中，这样，就可以在频繁更新时获得更好的性能。
关于 per-cpu 变量使用的例子可见于网络子系统中。内核维护者着大量计数器，这些计数器跟踪已接收到的各类数据包数量，而这些计数器每秒可能被更新上千次。网络子系统的开发者将这些统计用的计数器放在了 per-cpu 变量中，这样，他们就不需要处理缓存和锁定问题，而更新可在不用锁的情况下快速完成。在用户空间偶尔请求这些计数器的值时，只需将每个处理器的版本相加并返回合计值即可。

示例

操作 per-cpu 变量需要使用特定的函数，本文只作为一个引文，不进行每个函数的详细介绍，需要的话请自行查阅，以下直接给出一个 per-cpu 变量的使用示例
hello.c

#include <linux/delay.h>
#include <linux/fs.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>

#define MAJOR_NUM 231
#define DEVICE_NAME "hello"

DEFINE_PER_CPU(long, g_usage) = 0; // 定义一个per-cpu变量

int open(struct inode *p_inode, struct file *p_file)
{
	printk(KERN_ALERT DEVICE_NAME " hello open.\n");
	return (0);
}

ssize_t write(struct file *p_file, const char __user *p_buffer, size_t n_count, loff_t *p_offset)
{
	printk(KERN_ALERT DEVICE_NAME " hello write.\n");
	return (0);
}

int close(struct inode *p_inode, struct file *p_file)
{
	int i = 0;
	unsigned long base_addr = 0;
	unsigned long offset = 0;
	long *p_usage = NULL;
	long usage_sum = 0;

	offset = (unsigned long)(&g_usage);
	for_each_online_cpu(i)
	{
		base_addr = __per_cpu_offset[i];
		p_usage = (long *)(base_addr + offset);
		usage_sum += (*p_usage);
		printk(KERN_ALERT DEVICE_NAME " p_usage = %lx, *p_usage = %ld\n", (unsigned long)p_usage, *p_usage);
	}
	printk(KERN_ALERT DEVICE_NAME " %ld\n", usage_sum);
	return (0);
}

long ioctl(struct file *p_file, unsigned int uiCmd, unsigned long ulArg)
{
	long *p_usage = NULL;
	/* printk( KERN_ALERT DEVICE_NAME ": p_usage = 0x%lx %lx %ld", (unsigned long) p_usage, (unsigned long)
	 * (&g_usage), (*p_usage) ); */
	preempt_disable();
	p_usage = this_cpu_ptr((long *)(&g_usage));
	(*p_usage)++;
	preempt_enable();
	return (0);
}

struct file_operations hello_flops = {
	.open = open,
	.write = write,
	.release = close,
	.unlocked_ioctl = ioctl
};

static int __init hello_init(void)
{
	int ret;

	ret = register_chrdev(MAJOR_NUM, DEVICE_NAME, &hello_flops);
	if (ret < 0) {
		printk(KERN_ALERT DEVICE_NAME " can't register major number.\n");
		return (ret);
	}
	printk(KERN_ALERT DEVICE_NAME " initialized.\n");
	return (0);
}

static void __exit hello_exit(void)
{
	printk(KERN_ALERT DEVICE_NAME " removed.\n");
	unregister_chrdev(MAJOR_NUM, DEVICE_NAME);
}

module_init(hello_init);
module_exit(hello_exit);
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Sean Depp");

test.c

#include <stdio.h>
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <string.h>
#include <unistd.h>
#include <sys/ioctl.h>
#include <sys/stat.h>

#define NUM_THREADS 20

int fd = 0; // 文件描述符

// 多线程调用函数
void *pthread_fx(void *args)
{
	ioctl(fd, 1, 0);
}

int main()
{
	int ret = 0;

	// 打开文件
	if ((fd = open("/dev/hello", O_RDWR)) < 0) {
		printf("%s\n", strerror(errno));
		return -1;
	}

	// 开启多线程
	pthread_t tids[NUM_THREADS];
	for (int i = 0; i < NUM_THREADS; i++) {
		ret = pthread_create(&tids[i], NULL, pthread_fx, NULL);
		if (ret != 0) {
			printf("pthread_create error: error_code = %d\n", ret);
		}
	}

	// 回收线程资源
	for (int i = 0; i < NUM_THREADS; ++i) {
		pthread_join(tids[i], NULL);
	}

	// 关闭文件
	close(fd);

	return 0;
}

Makefile

KERNELDIR ?= /lib/modules/$(shell uname -r)/build
PWD       := $(shell pwd)

obj-m	:=hello.o

all:
	make -C $(KERNELDIR) M=$(PWD) modules

clean:
	rm -rf *.o *~ core .depend .*.cmd *.ko *.mod.* .tmp_versions *.mod *.order *.symvers *.dwo

test:
	gcc test.c -o test.out -lpthread

insmod:
	sudo insmod hello.ko
	sudo mknod /dev/hello c 231 0

rmmod:
	sudo rm /dev/hello
	sudo rmmod hello

编译内核模块

$ make
make -C /lib/modules/5.15.0-70-generic/build M=/home/liyongjun/project/c/C_study/kernel/per-cpu modules
make[1]: 进入目录“/usr/src/linux-headers-5.15.0-70-generic”
  CC [M]  /home/liyongjun/project/c/C_study/kernel/per-cpu/hello.o
  MODPOST /home/liyongjun/project/c/C_study/kernel/per-cpu/Module.symvers
  CC [M]  /home/liyongjun/project/c/C_study/kernel/per-cpu/hello.mod.o
  LD [M]  /home/liyongjun/project/c/C_study/kernel/per-cpu/hello.ko
  BTF [M] /home/liyongjun/project/c/C_study/kernel/per-cpu/hello.ko
Skipping BTF generation for /home/liyongjun/project/c/C_study/kernel/per-cpu/hello.ko due to unavailability of vmlinux
make[1]: 离开目录“/usr/src/linux-headers-5.15.0-70-generic”

编译 app

$ make test
gcc test.c -o test.out -lpthread

安装内核模块，并创建设备节点文件

$ make insmod 
sudo insmod hello.ko
sudo mknod /dev/hello c 231 0

运行 app

$ sudo ./test.out

dmesg 查看

$ dmesg | tail -n 7
[282663.589797] hello initialized.
[282726.834548] hello hello open.
[282726.835184] hello p_usage = ffff9053bea33010, *p_usage = 4
[282726.835189] hello p_usage = ffff9053beab3010, *p_usage = 5
[282726.835191] hello p_usage = ffff9053beb33010, *p_usage = 8
[282726.835192] hello p_usage = ffff9053bebb3010, *p_usage = 3
[282726.835192] hello 20

程序说明：
应用程序创建了 20 个线程，这 20 个线程终归要在 4 个核上运行，最终在 4 个核上执行次数计数 4、5、8、3，总和等于 20。