Memcache源码阅读（8）---多线程

我看的源码版本是1.2.4

前面已经将memcached的主要内容讨论完了。这里来说说memcached怎么使用多线程的吧。memcached的线程分为两种，一个主线程，另外的是工作线程。一个主线程负责接收用户的请求，接收到请求后将这个请求递交给多个工作线程的其中一个。每个工作线程都有一个连接队列管理着主线程递交的连接。

线程初始化

还是从线程的初始化开始。初始化一些锁，然后初始化工作线程，在主线程和工作线程之间建立管道进行通信。使用条件变量等待多个线程初始完成后函数返回。

void thread_init(int nthreads, struct event_base *main_base) {
    int         i;

    pthread_mutex_init(&cache_lock, NULL);
    pthread_mutex_init(&conn_lock, NULL);
    pthread_mutex_init(&slabs_lock, NULL);
    pthread_mutex_init(&stats_lock, NULL);

    pthread_mutex_init(&init_lock, NULL);
    pthread_cond_init(&init_cond, NULL);

    pthread_mutex_init(&cqi_freelist_lock, NULL);
    cqi_freelist = NULL;

    threads = malloc(sizeof(LIBEVENT_THREAD) * nthreads);
    if (! threads) {
        perror("Can't allocate thread descriptors");
        exit(1);
    }

    threads[0].base = main_base;
    threads[0].thread_id = pthread_self();

    for (i = 0; i < nthreads; i++) {
        int fds[2];
        if (pipe(fds)) {
            perror("Can't create notify pipe");
            exit(1);
        }

        threads[i].notify_receive_fd = fds[0];
        threads[i].notify_send_fd = fds[1];
    //设置线程的管道事件及其处理方式
    setup_thread(&threads[i]);
    }

    /* Create threads after we've done all the libevent setup. */
    for (i = 1; i < nthreads; i++) {
            //初始化工作线程，让工作线程处于livevent监听事件发生的状态
        create_worker(worker_libevent, &threads[i]);
    }

    /* Wait for all the threads to set themselves up before returning. */
    pthread_mutex_lock(&init_lock);
    init_count++; /* main thread */
    while (init_count < nthreads) {
        pthread_cond_wait(&init_cond, &init_lock);
    }
    pthread_mutex_unlock(&init_lock);
}
//为工作线程设置事件处理函数
static void setup_thread(LIBEVENT_THREAD *me) {
    if (! me->base) {
        me->base = event_init();
        if (! me->base) {
            fprintf(stderr, "Can't allocate event base\n");
            exit(1);
        }
    }

    /* Listen for notifications from other threads */
    //主线程接收到连接之后，会给该线程的pipe写一个字，该工作线程就会触发事件，
    //调用thread_libevent_process从工作队列中取连接处理。
    event_set(&me->notify_event, me->notify_receive_fd,
              EV_READ | EV_PERSIST, thread_libevent_process, me);
    event_base_set(me->base, &me->notify_event);

    if (event_add(&me->notify_event, 0) == -1) {
        fprintf(stderr, "Can't monitor libevent notify pipe\n");
        exit(1);
    }

    cq_init(&me->new_conn_queue);
}
static void thread_libevent_process(int fd, short which, void *arg) {
    LIBEVENT_THREAD *me = arg;
    CQ_ITEM *item;
    char buf[1];

    if (read(fd, buf, 1) != 1)
        if (settings.verbose > 0)
            fprintf(stderr, "Can't read from libevent pipe\n");

    item = cq_peek(&me->new_conn_queue);

    if (NULL != item) {
            //对这个连接建立事件处理函数，这个函数第三篇文章有讨论过。
        conn *c = conn_new(item->sfd, item->init_state, item->event_flags,
                           item->read_buffer_size, item->is_udp, me->base);
        if (c == NULL) {
            if (item->is_udp) {
                fprintf(stderr, "Can't listen for events on UDP socket\n");
                exit(1);
            } else {
                if (settings.verbose > 0) {
                    fprintf(stderr, "Can't listen for events on fd %d\n",
                        item->sfd);
                }
                close(item->sfd);
            }
        }
        cqi_free(item);
    }
}

static void create_worker(void *(*func)(void *), void *arg) {
    pthread_t       thread;
    pthread_attr_t  attr;
    int             ret;

    pthread_attr_init(&attr);

    if ((ret = pthread_create(&thread, &attr, func, arg)) != 0) {
        fprintf(stderr, "Can't create thread: %s\n",
                strerror(ret));
        exit(1);
    }
}
//让工作线程处于livevent监听事件发生的状态
static void *worker_libevent(void *arg) {
    LIBEVENT_THREAD *me = arg;

    /* Any per-thread setup can happen here; thread_init() will block until
     * all threads have finished initializing.
     */

    pthread_mutex_lock(&init_lock);
    init_count++;
    pthread_cond_signal(&init_cond);
    pthread_mutex_unlock(&init_lock);

    return (void*) event_base_loop(me->base, 0);
}

多线程操作

这个版本的锁粒度做得比较粗，很多访问数据的操作都是使用cache_lock这个锁，多线程操作知识在那个函数前后添加加锁解锁的操作。比如说更新item的操作。

//多线程版本
# define item_update(x)              mt_item_update(x)
void mt_item_update(item *item) {
    pthread_mutex_lock(&cache_lock);
    do_item_update(item);
    pthread_mutex_unlock(&cache_lock);
}

//单线程版本
# define item_update(x)              do_item_update(x)

到这里memcached的源码就讨论完了。