skynet源码阅读<6>--线程调度

最新推荐文章于 2023-08-30 16:32:04 发布

weixin_33872566

最新推荐文章于 2023-08-30 16:32:04 发布

阅读量111

点赞数

文章标签： lua 数据结构与算法运维

原文链接：http://www.cnblogs.com/Jackie-Snow/p/6907185.html

版权

相比于上节我们提到的协程调度，skynet的线程调度从逻辑流程上来看要简单很多。下面我们就来具体做一分析。首先自然是以skynet_start.c为入口：

 1 static void
 2 start(int thread) {
 3     pthread_t pid[thread+3];
 4 
 5     struct monitor *m = skynet_malloc(sizeof(*m));
 6     memset(m, 0, sizeof(*m));
 7     m->count = thread;
 8     m->sleep = 0;
 9 
10     m->m = skynet_malloc(thread * sizeof(struct skynet_monitor *));
11     int i;
12     for (i=0;i<thread;i++) {
13         m->m[i] = skynet_monitor_new();
14     }
15     if (pthread_mutex_init(&m->mutex, NULL)) {
16         fprintf(stderr, "Init mutex error");
17         exit(1);
18     }
19     if (pthread_cond_init(&m->cond, NULL)) {
20         fprintf(stderr, "Init cond error");
21         exit(1);
22     }
23 
24     create_thread(&pid[0], thread_monitor, m);
25     create_thread(&pid[1], thread_timer, m);
26     create_thread(&pid[2], thread_socket, m);
27 
28     static int weight[] = { 
29         -1, -1, -1, -1, 0, 0, 0, 0,
30         1, 1, 1, 1, 1, 1, 1, 1, 
31         2, 2, 2, 2, 2, 2, 2, 2, 
32         3, 3, 3, 3, 3, 3, 3, 3, };
33     struct worker_parm wp[thread];
34     for (i=0;i<thread;i++) {
35         wp[i].m = m;
36         wp[i].id = i;
37         if (i < sizeof(weight)/sizeof(weight[0])) {
38             wp[i].weight= weight[i];
39         } else {
40             wp[i].weight = 0;
41         }
42         create_thread(&pid[i+3], thread_worker, &wp[i]);
43     }
44 
45     for (i=0;i<thread+3;i++) {
46         pthread_join(pid[i], NULL); 
47     }
48 
49     free_monitor(m);
50 }

先是创建monitor、timer、socket三个线程分别执行监控、计时器、网络处理的工作，接着创建thread个用户线程，设置权重weight并开始执行。看下thread_worker里面做了什么：

 1 static void *
 2 thread_worker(void *p) {
 3     struct worker_parm *wp = p;
 4     int id = wp->id;
 5     int weight = wp->weight;
 6     struct monitor *m = wp->m;
 7     struct skynet_monitor *sm = m->m[id];
 8     skynet_initthread(THREAD_WORKER);
 9     struct message_queue * q = NULL;
10     while (!m->quit) {
11         q = skynet_context_message_dispatch(sm, q, weight);
12         if (q == NULL) {
13             if (pthread_mutex_lock(&m->mutex) == 0) {
14                 ++ m->sleep;
15                 // "spurious wakeup" is harmless,
16                 // because skynet_context_message_dispatch() can be call at any time.
17                 if (!m->quit)
18                     pthread_cond_wait(&m->cond, &m->mutex);
19                 -- m->sleep;
20                 if (pthread_mutex_unlock(&m->mutex)) {
21                     fprintf(stderr, "unlock mutex error");
22                     exit(1);
23                 }
24             }
25         }
26     }
27     return NULL;
28 }

每一个工作线程，先是分发消息，并获得下一次要分发的消息队列。如果没有的话就休息一会儿，等待合适的时机被唤醒。每次socket线程中有新的消息到来，或是timer-update时就唤醒所有休息在m->cond上的线程，继续消息分发。看一下具体的分发过程：

 1 struct message_queue * 
 2 skynet_context_message_dispatch(struct skynet_monitor *sm, struct message_queue *q, int weight) {
 3     if (q == NULL) {
 4         q = skynet_globalmq_pop();
 5         if (q==NULL)
 6             return NULL;
 7     }
 8 
 9     uint32_t handle = skynet_mq_handle(q);
10 
11     struct skynet_context * ctx = skynet_handle_grab(handle);
12     if (ctx == NULL) {
13         struct drop_t d = { handle };
14         skynet_mq_release(q, drop_message, &d);
15         return skynet_globalmq_pop();
16     }
17 
18     int i,n=1;
19     struct skynet_message msg;
20 
21     for (i=0;i<n;i++) {
22         if (skynet_mq_pop(q,&msg)) {
23             skynet_context_release(ctx);
24             return skynet_globalmq_pop();
25         } else if (i==0 && weight >= 0) {
26             n = skynet_mq_length(q);
27             n >>= weight;
28         }
29         int overload = skynet_mq_overload(q);
30         if (overload) {
31             skynet_error(ctx, "May overload, message queue length = %d", overload);
32         }
33 
34         skynet_monitor_trigger(sm, msg.source , handle);
35 
36         if (ctx->cb == NULL) {
37             skynet_free(msg.data);
38         } else {
39             dispatch_message(ctx, &msg);
40         }
41 
42         skynet_monitor_trigger(sm, 0,0);
43     }
44 
45     assert(q == ctx->queue);
46     struct message_queue *nq = skynet_globalmq_pop();
47     if (nq) {
48         // If global mq is not empty , push q back, and return next queue (nq)
49         // Else (global mq is empty or block, don't push q back, and return q again (for next dispatch)
50         skynet_globalmq_push(q);
51         q = nq;
52     } 
53     skynet_context_release(ctx);
54 
55     return q;
56 }

拿到当前消息队列q对应的handle和context，接着便从q中连续取出n个消息通过dispatch_message分发出去。权重越大的线程，当前可分发的消息数n就越大。如果在n范围内q的消息分发完了，那么就从全局队列globalmq中取出下一个消息队列q，返回等待下一次执行；如果n范围内q依然有消息剩余，那么就把当前消息队列q扔回到global_mq中去，并拿到新的消息队列nq返回，这样每个消息队列都会相对公平地得到执行。不同的线程只是在从全局队列global_mq拿消息队列q时对global_mq加锁，或者在向q中推入消息（比如向skynet-context的q发送消息），以及这里取出消息执行时需要对q加锁。
具体的q或global_mq是按照链表实现的，这里就不废话了。最后看下消息分发的处理：

 1 static void
 2 dispatch_message(struct skynet_context *ctx, struct skynet_message *msg) {
 3     assert(ctx->init);
 4     CHECKCALLING_BEGIN(ctx)
 5     pthread_setspecific(G_NODE.handle_key, (void *)(uintptr_t)(ctx->handle));
 6     int type = msg->sz >> MESSAGE_TYPE_SHIFT;
 7     size_t sz = msg->sz & MESSAGE_TYPE_MASK;
 8     if (ctx->logfile) {
 9         skynet_log_output(ctx->logfile, msg->source, type, msg->session, msg->data, sz);
10     }
11     ++ctx->message_count;
12     int reserve_msg;
13     if (ctx->profile) {
14         ctx->cpu_start = skynet_thread_time();
15         reserve_msg = ctx->cb(ctx, ctx->cb_ud, type, msg->session, msg->source, msg->data, sz);
16         uint64_t cost_time = skynet_thread_time() - ctx->cpu_start;
17         ctx->cpu_cost += cost_time;
18     } else {
19         reserve_msg = ctx->cb(ctx, ctx->cb_ud, type, msg->session, msg->source, msg->data, sz);
20     }
21     if (!reserve_msg) {
22         skynet_free(msg->data);
23     }
24     CHECKCALLING_END(ctx)
25 }

关键就在于ctx->cb这个函数指针的调用。你肯定会好奇，这个回调是如何进入LUA代码的呢？回想skynet.lua中skynet.start(f)函数，其调用了c.callback将回调f注册到skynet-os中。因此就让我们回到skynet-lua.c中，看一看lcallback函数的处理：

 1 static int
 2 lcallback(lua_State *L) {
 3     struct skynet_context * context = lua_touserdata(L, lua_upvalueindex(1));
 4     int forward = lua_toboolean(L, 2);
 5     luaL_checktype(L,1,LUA_TFUNCTION);
 6     lua_settop(L,1);
 7     lua_rawsetp(L, LUA_REGISTRYINDEX, _cb);
 8 
 9     lua_rawgeti(L, LUA_REGISTRYINDEX, LUA_RIDX_MAINTHREAD);
10     lua_State *gL = lua_tothread(L,-1);
11 
12     if (forward) {
13         skynet_callback(context, gL, forward_cb);
14     } else {
15         skynet_callback(context, gL, _cb);
16     }
17 
18     return 0;
19 }

这里重点看下lua_rawsetp(L, LUA_REGISTRYINDEX, _cb)。此时栈顶为LUA_TFUNCTION元素f，这句的作用是从索引LUA_REGISTERINDEX处取出t，在这里也就是全局注册表，然后设置t[_cb]=f并弹出栈顶元素（此函数不触发__index元方法）。这样就将_cb与f之间关联起来了。最后调用skynet_callback将_cb设置到context.cb元素，而虚拟机L也设置到context.ud元素。了解了lcallback的注册过程，我们回到_cb函数，看看具体调用时发生了什么事情：

 1 static int
 2 _cb(struct skynet_context * context, void * ud, int type, int session, uint32_t source, const void * msg, size_t sz) {
 3     lua_State *L = ud;
 4     int trace = 1;
 5     int r;
 6     int top = lua_gettop(L);
 7     if (top == 0) {
 8         lua_pushcfunction(L, traceback);
 9         lua_rawgetp(L, LUA_REGISTRYINDEX, _cb);
10     } else {
11         assert(top == 2);
12     }
13     lua_pushvalue(L,2);
14 
15     lua_pushinteger(L, type);
16     lua_pushlightuserdata(L, (void *)msg);
17     lua_pushinteger(L,sz);
18     lua_pushinteger(L, session);
19     lua_pushinteger(L, source);
20 
21     r = lua_pcall(L, 5, 0 , trace);
22 
23     if (r == LUA_OK) {
24         return 0;
25     }
26 }