vpp中的协程实现

        最近在看 vpp 的源码，发现里面用了大量的 setjmp 和 longjmp，在逐渐深入阅读代码的过程中才意识到这就是所谓的协程呀。花了一些时间将其中的关键代码抽出来进行调试，收获不少，想要学习协程的同学可以做个参考。
        下面这个关于协程的 demo 是基于 x86 架构的，一共涉及四个文件，分别是

• longjmp.h  // 声明 jmp 相关函数的原型
• longjmp.S   // 使用 x86 汇编代码实现 jmp 相关函数
• main.c  // 使用 jmp 相关函数实现协程和进行测试
• Makefile

        longjmp.h 源码

#ifndef included_clib_longjmp_h
#define included_clib_longjmp_h

#define CLIB_ARCH_LONGJMP_REGS (22)

typedef unsigned long u64;
typedef u64 uword;

typedef struct
{
  uword regs[CLIB_ARCH_LONGJMP_REGS];
} clib_longjmp_t __attribute__ ((aligned (16)));

/* Return given value to saved context. */
void clib_longjmp (clib_longjmp_t * save, uword return_value);

/* Save context.  Returns given value if jump is not taken;
   otherwise returns value from clib_longjmp if long jump is taken. */
uword clib_setjmp (clib_longjmp_t * save, uword return_value_not_taken);

/* Call function on given stack. */
uword clib_calljmp (uword (*func) (uword func_arg),
		    uword func_arg, void *stack);

#endif /* included_clib_longjmp_h */

/*
 * fd.io coding-style-patch-verification: ON
 *
 * Local Variables:
 * eval: (c-set-style "gnu")
 * End:
 */

        longjmp.S 源码

	.global clib_setjmp
	.align 4
	.type clib_setjmp, @function
clib_setjmp:
	movq %rbx, 8*0(%rdi)
	movq %rbp, 8*1(%rdi)
	movq %r12, 8*2(%rdi)
	movq %r13, 8*3(%rdi)
	movq %r14, 8*4(%rdi)
	movq %r15, 8*5(%rdi)

	/* Save SP after return. */
	leaq 8(%rsp), %rdx
	movq %rdx, 8*6(%rdi)
	
	/* Save PC we are returning to from stack frame. */
	movq 0(%rsp), %rax
	movq %rax, 8*7(%rdi)
	
	/* Give back user's return value. */
	movq %rsi, %rax
	ret
	
	.global clib_longjmp
	.align 4
	.type clib_longjmp, @function
clib_longjmp:	
        /* Restore regs. */
	movq 8*0(%rdi), %rbx
	movq 8*1(%rdi), %rbp
	movq 8*2(%rdi), %r12
	movq 8*3(%rdi), %r13
	movq 8*4(%rdi), %r14
	movq 8*5(%rdi), %r15
	movq 8*6(%rdi), %rsp
	movq 8*7(%rdi), %rdx
	
	/* Give back user's return value. */
	movq %rsi, %rax
	
	/* Away we go. */
	jmpq *%rdx	
	
	.global clib_calljmp
	.align 4
	.type clib_calljmp, @function
clib_calljmp:
	/* Make sure stack is 16-byte aligned. */
	movq %rdx, %rax
	andq $0xf, %rax
	subq %rax, %rdx
	
	/* Get return address. */
	pop %rax
	
	/* Switch to new stack. */
	xchgq %rsp, %rdx
	
	/* Save return address on new stack. */
	push %rax
	
	/* Save old stack pointer on new stack. */
	push %rdx
	
	/* Get function. */
	movq %rdi, %rdx
	
	/* Move argument into place. */
	movq %rsi, %rdi
	
	/* Away we go. */
	callq *%rdx
	
	/* Switch back to old stack. */
	movq 8(%rsp), %rdx
	movq 0(%rsp), %rcx
	xchgq %rcx, %rsp
	
	/* Return to caller. */
	jmpq *%rdx

        main.c 源码

#include <assert.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <time.h>
#include <stdint.h>
#include <sys/time.h>
#include <unistd.h>
#include "longjmp.h"

#define always_inline static inline __attribute__ ((__always_inline__))

#define RETURN_SETJMP_MAGIC 0x0930    
#define RETURN_LONGJMP_MAGIC 0x0931

#define RESUME_SETJMP_MAGIC 0x1112
#define RESUME_LONGJMP_MAGIC 0x1113

typedef double f64;

typedef uword (*worker_func_t)(uword func_arg);

struct process_context {
	f64 interval;
	clib_longjmp_t return_longjmp;
 	/* Where to longjmp to resume node after suspend. */
	clib_longjmp_t resume_longjmp;
	int process_index;
	void *stack;
	int stack_size;
	worker_func_t function;
	f64 wait_left;
	f64 wakeup_time;
	const char *name;
};

uword oam_process(uword arg);

uword dhcp_process(uword arg);

struct function_register {
	worker_func_t function;
	const char *name;
};

struct function_register worker_funcs[] = {
	{oam_process, "oam_process"},
	{dhcp_process, "dhcp_process"},
};

#define WORKER_NUM  (sizeof(worker_funcs) / sizeof(struct function_register))

struct process_context workers[WORKER_NUM];

always_inline int vlib_process_suspend_time_is_zero(f64 dt);

always_inline int vlib_process_suspend_time_is_zero(f64 dt)
{
	return dt < 10e-6;
}

int init_workers(void)
{
	int i;

	memset(workers, 0, sizeof(workers));

	for (i = 0; i < WORKER_NUM; ++i) {
		workers[i].interval = (i + 1);
		workers[i].process_index = i;
		workers[i].stack_size = (1 << 16);
		workers[i].stack = malloc(sizeof(char) * workers[i].stack_size);
	    if (workers[i].stack == NULL) {
			printf("alloc stack for process failed!\n");
			return -1;
		}
		memset(workers[i].stack, 0, sizeof(char) * workers[i].stack_size);
		workers[i].function = worker_funcs[i].function;
		workers[i].name = worker_funcs[i].name;
	}

	return 0;
}

always_inline f64 vlib_time_now(void)
{
    struct timespec ts;
 
    clock_gettime(CLOCK_MONOTONIC, &ts);
 
    return (double)ts.tv_sec + (double)ts.tv_nsec / 1000000000.0;
}

always_inline f64 get_now(void)
{
	struct timeval t;

    gettimeofday(&t, NULL);

   	return (f64)t.tv_sec + (f64)t.tv_usec / 1000000.0;
}

always_inline f64 vlib_process_wait_for_event_or_clock(
        struct process_context *worker, f64 dt);

always_inline f64 vlib_process_wait_for_event_or_clock(
        struct process_context *worker, f64 dt)
{
    f64 wakeup_time;
	int r;

	if (vlib_process_suspend_time_is_zero(dt))
		return dt; 

	wakeup_time = get_now() + dt;
	r = clib_setjmp(&worker->resume_longjmp, RESUME_SETJMP_MAGIC);
	if (r == RESUME_SETJMP_MAGIC) {
		worker->wakeup_time = wakeup_time;
		worker->wait_left = dt;
		clib_longjmp(&worker->return_longjmp, RETURN_LONGJMP_MAGIC);
	}
	assert(worker->wakeup_time == wakeup_time);

	return wakeup_time - get_now();
}

int ts_to_str(uint64_t ts, char *buf, int max_len)
{
    time_t t = (time_t)ts;
    const char *format = "%Y-%m-%d %H:%M:%S";
    struct tm lt;
    int ret = 0;

    (void) localtime_r(&t, &lt);
    if ((ret = strftime(buf, max_len, format, &lt)) == 0) {
        return sprintf(buf, "unknown");
    }

    return ret;
}

uword oam_process(uword arg)
{
	f64 now, dt;
	int cnt = 0;
	char time_str[32];
	struct process_context *worker;

	worker = (struct process_context *)arg;
	dt = worker->interval;
	while (1) {
		dt = vlib_process_wait_for_event_or_clock(worker, dt);
		worker->wait_left = dt;
		if (!vlib_process_suspend_time_is_zero(dt))
			continue;

		now = get_now();
		ts_to_str(now, time_str, sizeof(time_str));
		printf("[%s][%s] worker %d: begin to recv packet ...\n", __func__,
				time_str, worker->process_index);

		/* recv one packet */

		/* handle this packet */
		
		now = get_now();
		ts_to_str(now, time_str, sizeof(time_str));
		printf("[%s][%s] worker %d: handle %d-th packet done!\n", __func__,
				time_str, worker->process_index, cnt);

		dt = worker->interval;
		cnt++;
	}

	return 0;
}

uword dhcp_process(uword arg)
{
	f64 now, dt;
	int cnt = 0;
	char time_str[32];
	struct process_context *worker;

	worker = (struct process_context *)arg;
	dt = worker->interval;
	while (1) {
		dt = vlib_process_wait_for_event_or_clock(worker, dt);
		worker->wait_left = dt;
		if (!vlib_process_suspend_time_is_zero(dt))
			continue;

		now = get_now();
		ts_to_str(now, time_str, sizeof(time_str));
		printf("[%s][%s] worker %d: begin to recv packet ...\n", __func__,
				time_str, worker->process_index);

		/* recv one packet */

		/* handle this packet */
		
		now = get_now();
		ts_to_str(now, time_str, sizeof(time_str));
		printf("[%s][%s] worker %d: handle %d-th packet done!\n", __func__,
				time_str, worker->process_index, cnt);

		dt = worker->interval;
		cnt++;
	}

	return 0;
}

int resume_process(struct process_context *worker)
{
	int r;

	r = clib_setjmp(&worker->return_longjmp, RETURN_SETJMP_MAGIC);
	if (r == RETURN_SETJMP_MAGIC) {
		clib_longjmp(&worker->resume_longjmp, RESUME_LONGJMP_MAGIC);
	}

	return 0;
}

int start_process(struct process_context *worker)
{
	int r;

	r = clib_setjmp(&worker->return_longjmp, RETURN_SETJMP_MAGIC); 
	if (r == RETURN_SETJMP_MAGIC) {
		r = clib_calljmp(worker->function, (uword)worker,
				worker->stack + worker->stack_size);
	}

	return r;
}

int main(void)
{
	int i;
	
	if (init_workers() < 0) {
		printf("init workers failed!\n");
		return -1;
	}

	for (i = 0; i < WORKER_NUM; ++i) {
		start_process(&workers[i]);
	}

	while (1) {
		for (i = 0; i < WORKER_NUM; ++i) {
			resume_process(&workers[i]);
		}
        /* do something */
	}

	return 0;
}

        Makefile 源码

all: main

main: main.c longjmp.S 
	gcc -g -O2 -DFORTIFY_SOURCE=2 -fstack-protector -fPIC -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast  -Wno-address-of-packed-member $^ -o $@

clean:
	rm main

        运行效果：

# make
gcc -g -O2 -DFORTIFY_SOURCE=2 -fstack-protector -fPIC -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast  -Wno-address-of-packed-member main.c longjmp.S -o main
# ./main 
[oam_process][2021-11-23 19:22:26] worker 0: begin to recv packet ...
[oam_process][2021-11-23 19:22:26] worker 0: handle 0-th packet done!
[oam_process][2021-11-23 19:22:27] worker 0: begin to recv packet ...
[oam_process][2021-11-23 19:22:27] worker 0: handle 1-th packet done!
[dhcp_process][2021-11-23 19:22:27] worker 1: begin to recv packet ...
[dhcp_process][2021-11-23 19:22:27] worker 1: handle 0-th packet done!
[oam_process][2021-11-23 19:22:28] worker 0: begin to recv packet ...
[oam_process][2021-11-23 19:22:28] worker 0: handle 2-th packet done!
[oam_process][2021-11-23 19:22:30] worker 0: begin to recv packet ...
[oam_process][2021-11-23 19:22:30] worker 0: handle 3-th packet done!
[dhcp_process][2021-11-23 19:22:30] worker 1: begin to recv packet ...
[dhcp_process][2021-11-23 19:22:30] worker 1: handle 1-th packet done!
[oam_process][2021-11-23 19:22:31] worker 0: begin to recv packet ...
[oam_process][2021-11-23 19:22:31] worker 0: handle 4-th packet done!
[oam_process][2021-11-23 19:22:32] worker 0: begin to recv packet ...
[oam_process][2021-11-23 19:22:32] worker 0: handle 5-th packet done!
[dhcp_process][2021-11-23 19:22:32] worker 1: begin to recv packet ...
[dhcp_process][2021-11-23 19:22:32] worker 1: handle 2-th packet done!
[oam_process][2021-11-23 19:22:33] worker 0: begin to recv packet ...
[oam_process][2021-11-23 19:22:33] worker 0: handle 6-th packet done!
[dhcp_process][2021-11-23 19:22:35] worker 1: begin to recv packet ...
[dhcp_process][2021-11-23 19:22:35] worker 1: handle 3-th packet done!
[oam_process][2021-11-23 19:22:35] worker 0: begin to recv packet ...
[oam_process][2021-11-23 19:22:35] worker 0: handle 7-th packet done!
[oam_process][2021-11-23 19:22:36] worker 0: begin to recv packet ...
[oam_process][2021-11-23 19:22:36] worker 0: handle 8-th packet done!
[dhcp_process][2021-11-23 19:22:37] worker 1: begin to recv packet ...
[dhcp_process][2021-11-23 19:22:37] worker 1: handle 4-th packet done!
[oam_process][2021-11-23 19:22:37] worker 0: begin to recv packet ...
[oam_process][2021-11-23 19:22:37] worker 0: handle 9-th packet done!
[oam_process][2021-11-23 19:22:39] worker 0: begin to recv packet ...
[oam_process][2021-11-23 19:22:39] worker 0: handle 10-th packet done!
[dhcp_process][2021-11-23 19:22:40] worker 1: begin to recv packet ...
[dhcp_process][2021-11-23 19:22:40] worker 1: handle 5-th packet done!