高性能协程库State Thread Library源代码剖析之二寻找入口

目前人类认知的所有现象，规律，定理都是有源可寻，有根可究的，唯一不知道根源的可能是谁创造了万有引力。对于软件工程师来说，一个软件系统生命开始的地方就在软件系统的入口，应用程序就是main函数，kernel就是kernel_entry。但是这里我们研究的是一个库，库自然是给应用程序使用的，应用程序在哪里呢，在example下。

lookupdns.c 查找域名的ip，可输入多个域名
st_init()
for (i = 1; i < argc; i++)
    st_thread_create(do_resolve, argv[i], 0, 0)
st_thread_exit(NULL);

就这几行代码， 包含了整个协程库的初始化，调度，结束。这里用线程替代协程描述

st_init()创建idle线程，和点火线程，点火线程一旦退出，就永远不再

 st_thread_create创建用户线程

st_thread_exit(NULL);才是真正调度的开始，调度完后，才退出

接口分析
st_init
typedef struct _st_eventsys_ops {
  const char *name;                          /* Name of this event system */
  int  val;                                  /* Type of this event system */
  int  (*init)(void);                        /* Initialization */
  void (*dispatch)(void);                    /* Dispatch function */
  int  (*pollset_add)(struct pollfd *, int); /* Add descriptor set */
  void (*pollset_del)(struct pollfd *, int); /* Delete descriptor set */
  int  (*fd_new)(int);                       /* New descriptor allocated */
  int  (*fd_close)(int);                     /* Descriptor closed */
  int  (*fd_getlimit)(void);                 /* Descriptor hard limit */
} _st_eventsys_t;

static _st_eventsys_t _st_poll_eventsys = {
    "poll",
    ST_EVENTSYS_POLL,
    _st_poll_init,
    _st_poll_dispatch,
    _st_poll_pollset_add,
    _st_poll_pollset_del,
    _st_poll_fd_new,
    _st_poll_fd_close,
    _st_poll_fd_getlimit
};

static _st_eventsys_t _st_select_eventsys = {
    "select",
    ST_EVENTSYS_SELECT,
    _st_select_init,
    _st_select_dispatch,
    _st_select_pollset_add,
    _st_select_pollset_del,
    _st_select_fd_new,
    _st_select_fd_close,
    _st_select_fd_getlimit
};
_st_active_count代表是否初始化过
st_set_eventsys(ST_EVENTSYS_DEFAULT);选择_st_select_eventsys
_st_io_init
    rlim.rlim_max = FD_SETSIZE 设置打开的最大文件描述符数量，select有限制，pool无
    赋值给_st_osfd_limit

typedef struct _st_vp {
  _st_thread_t *idle_thread;  /* Idle thread for this vp */
  st_utime_t last_clock;      /* The last time we went into vp_check_clock() */

  _st_clist_t run_q;          /* run queue for this vp */
  _st_clist_t io_q;           /* io queue for this vp */
  _st_clist_t zombie_q;       /* zombie queue for this vp */
#ifdef DEBUG
  _st_clist_t thread_q;       /* all threads of this vp */
#endif
  int pagesize;

  _st_thread_t *sleep_q;      /* sleep queue for this vp */
  int sleepq_size;          /* number of threads on sleep queue */

#ifdef ST_SWITCH_CB 直接定义了
  st_switch_cb_t switch_out_cb;    /* called when a thread is switched out */
  st_switch_cb_t switch_in_cb;    /* called when a thread is switched in */
#endif
} _st_vp_t;

struct _st_thread {
  int state;                  /* Thread's state */
  int flags;                  /* Thread's flags */

  void *(*start)(void *arg);  /* The start function of the thread */
  void *arg;                  /* Argument of the start function */
  void *retval;               /* Return value of the start function */

  _st_stack_t *stack;          /* Info about thread's stack */

  _st_clist_t links;          /* For putting on run/sleep/zombie queue */
  _st_clist_t wait_links;     /* For putting on mutex/condvar wait queue */
#ifdef DEBUG
  _st_clist_t tlink;          /* For putting on thread queue */
#endif

  st_utime_t due;             /* Wakeup time when thread is sleeping */
  _st_thread_t *left;         /* For putting in timeout heap */
  _st_thread_t *right;          /* -- see docs/timeout_heap.txt for details */
  int heap_index;

  void **private_data;        /* Per thread private data */

  _st_cond_t *term;           /* Termination condition variable for join */

  jmp_buf context;            /* Thread's context */
};

typedef struct _st_stack {
  _st_clist_t links;
  char *vaddr;                /* Base of stack's allocated memory */
  int  vaddr_size;            /* Size of stack's allocated memory */
  int  stk_size;              /* Size of usable portion of the stack */
  char *stk_bottom;           /* Lowest address of stack's usable portion */
  char *stk_top;              /* Highest address of stack's usable portion */
  void *sp;                   /* Stack pointer from C's point of view */
#ifdef __ia64__
  void *bsp;                  /* Register stack backing store pointer */
#endif
} _st_stack_t;

typedef struct _st_clist {
  struct _st_clist *next;
  struct _st_clist *prev;
} _st_clist_t;

typedef unsigned long long  st_utime_t;
typedef struct _st_cond {
  _st_clist_t wait_q;          /* Condition variable wait queue */
} _st_cond_t;
typedef struct _st_cond *   st_cond_t;

typedef void (*st_switch_cb_t)(void);

memset(&_st_this_vp, 0, sizeof(_st_vp_t));


ST_INIT_CLIST(&_ST_RUNQ); &_st_this_vp.run_q
ST_INIT_CLIST(&_ST_IOQ); &_st_this_vp.io_q
ST_INIT_CLIST(&_ST_ZOMBIEQ); &_st_this_vp.zombie_q
#ifdef DEBUG
ST_INIT_CLIST(&_ST_THREADQ); &_st_this_vp.thread_q
#endif

/* Global data */全局数据
_st_vp_t _st_this_vp;           /* This VP */
_st_thread_t *_st_this_thread;  /* Current thread */
int _st_active_count = 0;       /* Active thread count */

time_t _st_curr_time = 0;       /* Current time as returned by time(2) */
st_utime_t _st_last_tset;       /* Last time it was fetched */    


(*_st_eventsys->init)()
_st_poll_init
static struct _st_polldata {
    struct pollfd *pollfds; 预先分配64个
    int pollfds_size; ST_MIN_POLLFDS_SIZE 64
    int fdcnt; 0
} *_st_poll_data;
_st_poll_data分配空间

_st_select_init
static struct _st_seldata {
    fd_set fd_read_set, fd_write_set, fd_exception_set;
    int fd_ref_cnts[FD_SETSIZE][3];
    int maxfd;
} *_st_select_data;
分配空间
memset(_st_select_data, 0, sizeof(*_st_select_data));

_st_this_vp.pagesize = getpagesize(); 系统调用，得到页大小
_st_this_vp.last_clock = st_utime(); 得到当前时间

_st_this_vp.idle_thread = st_thread_create(_st_idle_thread_start, NULL, 0, 0);

stk_size = ST_DEFAULT_STACK_SIZE;
stk_size = ((stk_size + _ST_PAGE_SIZE - 1) / _ST_PAGE_SIZE) * _ST_PAGE_SIZE;
调整stk_size为页数的整数倍

stack = _st_stack_new(stk_size);
    先试着从_st_free_stacks链表里寻找（条件是ts->stk_size >= stack_size），_st_stack_free(_st_stack_t *ts)会往链表里存stack，位置是倒数第二
    分配一个_st_stack_t结构体
    extra = _st_randomize_stacks ? _ST_PAGE_SIZE : 0;
    #define REDZONE    _ST_PAGE_SIZE 栈与栈之间的空间
    ts->vaddr_size = stack_size + 2*REDZONE + extra;
    ts->vaddr = _st_new_stk_segment(ts->vaddr_size);    
        int mmap_flags = MAP_PRIVATE;
        mmap_flags |= MAP_ANON;
        
        分配内存（mmap可以分配内存，可以实现共享内存，可以高效读写文件）
        vaddr = mmap(NULL, size, PROT_READ | PROT_WRITE, mmap_flags, zero_fd, 0);
    ts->stk_size = stack_size;
    ts->stk_bottom = ts->vaddr + REDZONE; 底部留一页，顶部留一页
    ts->stk_top = ts->stk_bottom + stack_size;
    
定义了MD_STACK_GROWS_DOWN
sp = stack->stk_top;

sp = sp - (ST_KEYS_MAX * sizeof(void *)); 下移16个地址的长度
ptds = (void **) sp;
sp = sp - sizeof(_st_thread_t);
thread = (_st_thread_t *) sp; 保留_st_thread_t大小的

/* Make stack 64-byte aligned */
if ((unsigned long)sp & 0x3f) 6个1
    sp = sp - ((unsigned long)sp & 0x3f); 64位对齐
stack->sp = sp - _ST_STACK_PAD_SIZE;（MD_STACK_PAD_SIZE 128）

memset(thread, 0, sizeof(_st_thread_t));
memset(ptds, 0, ST_KEYS_MAX * sizeof(void *));

thread->private_data = ptds; 16个地址长度
thread->stack = stack; 指向整个stack（thread结构体也寄存在stack的vaddr）
thread->start = start; 线程函数
thread->arg = arg; 线程函数的参数

LINUX系统，amd64架构的定义
#define MD_STACK_GROWS_DOWN
#define MD_USE_BUILTIN_SETJMP

#ifndef JB_RSP
#define JB_RSP 6
#endif
#define MD_GET_SP(_t) (_t)->context[0].__jmpbuf[JB_RSP]

#if defined(MD_USE_BUILTIN_SETJMP) && !defined(USE_LIBC_SETJMP)
使用内置的
#define MD_SETJMP(env) _st_md_cxt_save(env)
#define MD_LONGJMP(env, val) _st_md_cxt_restore(env, val)

extern int _st_md_cxt_save(jmp_buf env);
extern void _st_md_cxt_restore(jmp_buf env, int val);
#else
...

#define _ST_INIT_CONTEXT MD_INIT_CONTEXT
#define MD_INIT_CONTEXT(_thread, _sp, _main) \
  ST_BEGIN_MACRO                             \
  if (MD_SETJMP((_thread)->context))         \
    _main();                                 \
  MD_GET_SP(_thread) = (long) (_sp);         \这一句必定执行
  ST_END_MACRO
 
 
setjmp，调用时，返回值是0; 从longjmp返回时，返回值由longjmp指定

_ST_INIT_CONTEXT(thread, stack->sp, _st_thread_main);
    #define MD_SETJMP(env) setjmp(env)
        setjmp((_thread)->context); 将此时的上下文保存到(_thread)->context，
        用自定义的汇编_st_md_cxt_save实现setjmp保存
        
        longjmp在这里返回时才执行
        _st_thread_main();
            _st_thread_t *thread = _ST_CURRENT_THREAD();
            /* Run thread main */
            thread->retval = (*thread->start)(thread->arg);
            这里调用的是_st_idle_thread_start
                _st_thread_t *me = _ST_CURRENT_THREAD();
                while (_st_active_count > 0) {
                    /* Idle vp till I/O is ready or the smallest timeout expired */
                    _ST_VP_IDLE(); (*_st_eventsys->dispatch)();
                        _st_poll_dispatch
                        _st_select_dispatch

                    /* Check sleep queue for expired threads */
                    _st_vp_check_clock();

                    me->state = _ST_ST_RUNNABLE;
                    _ST_SWITCH_CONTEXT(me);
                }
            
            /* All done, time to go away */
            st_thread_exit(thread->retval);
            
    #define MD_GET_SP(_t) (_t)->context[0].__jmpbuf[JB_RSP]
    amd64
    (_t)->context[0].__jmpbuf[JB_RSP] JB_RSP = 6
        thread->context[0].__jmpbuf[6] = (long) (_sp);

thread->term = _st_cond_t *链表
thread->state = _ST_ST_RUNNABLE;
_ST_ADD_RUNQ(thread); _st_this_vp.run_q

_st_this_vp.idle_thread->flags = _ST_FL_IDLE_THREAD;
_ST_DEL_RUNQ(_st_this_vp.idle_thread);
从runq队列中移除，但是没有销毁thread


初始化primordial（原始）线程
thread = (_st_thread_t *) calloc(1, sizeof(_st_thread_t) + (ST_KEYS_MAX * sizeof(void *)));
一个thread结构体和16个指针长度
thread->private_data = (void **) (thread + 1);
thread->state = _ST_ST_RUNNING;
thread->flags = _ST_FL_PRIMORDIAL; （设置标志，原始线程）
_ST_SET_CURRENT_THREAD(thread); 全局变量_st_this_thread
_st_active_count++;
原始线程没有加入runq队列

#define MD_LONGJMP(env, val) longjmp(env, val)
_ST_RESTORE_CONTEXT
#define _ST_RESTORE_CONTEXT(_thread)   \
    ST_BEGIN_MACRO                     \
    _ST_SET_CURRENT_THREAD(_thread);   \
    MD_LONGJMP((_thread)->context, 1); \
    ST_END_MACRO

_st_vp_schedule