libfuse__源码走读--多线程运行时

本文对源码的分析基于libfuse 3.12.0

一、fuse的多线程运行时

   fuse在运行时主要依靠fuse daemon与内核模块进行基于fuse协议的交互，libfuse通常采用多线程对这些请求进行处理，其中多线程运行时的主要实现在fuse_loop_mt.c中。

1.1 相关数据结构

//工作线程控制结构
struct fuse_worker {
	//双向循环链表
	struct fuse_worker *prev;
	struct fuse_worker *next;
	pthread_t thread_id;

	// We need to include fuse_buf so that we can properly free
	// it when a thread is terminated by pthread_cancel().
	struct fuse_buf fbuf;
	struct fuse_chan *ch;//与fuse内核连接的pipe的结构体
	struct fuse_mt *mt;//多线程控制结构
};

//多线程控制结构
struct fuse_mt {
	pthread_mutex_t lock;
	int numworker;//总线程数
	int numavail;//可用线程数
	struct fuse_session *se;//fuse会话结构体
	struct fuse_worker main;//工作主线程
	sem_t finish;//信号量
	int exit;
	int error;
	int clone_fd;//每个线程是否使用独立的设备fd
	int max_idle;//最大空闲线程数
	int max_threads;//最大线程数
};

//
struct fuse_chan {
	pthread_mutex_t lock;
	int ctr;//channel引用计数器
	int fd;//设备fd
};

1.2 从session启动函数看起

   fuse_session_loop_mt_31函数是进入fuse循环的主函数，该函数在用户进程和fuse内核模块的连接被中断时，会返回0。

int fuse_session_loop_mt_312(struct fuse_session *se, struct fuse_loop_config *config)
{
	int err;
	struct fuse_mt mt;
	struct fuse_worker *w;
	int created_config = 0;
	//检查fuse的config是否存在
	if (config) {	
		//检查config是否有效
		err = fuse_loop_cfg_verify(config);
		if (err)
			return err;
	} else {
		/* The caller does not care about parameters - use the default */
		config = fuse_loop_cfg_create();
		created_config = 1;
	}


	memset(&mt, 0, sizeof(struct fuse_mt));
	mt.se = se;
	mt.clone_fd = config->clone_fd;
	mt.error = 0;
	mt.numworker = 0;
	mt.numavail = 0;
	mt.max_idle = config->max_idle_threads;
	mt.max_threads = config->max_threads;
	mt.main.thread_id = pthread_self();
	mt.main.prev = mt.main.next = &mt.main;
	sem_init(&mt.finish, 0, 0);
	pthread_mutex_init(&mt.lock, NULL);

	pthread_mutex_lock(&mt.lock);
	err = fuse_loop_start_thread(&mt);//启动loop thread
	pthread_mutex_unlock(&mt.lock);
	if (!err) {
		/* sem_wait() is interruptible */
		while (!fuse_session_exited(se))//如果fuse会话没有退出，则阻塞在信号量上
			sem_wait(&mt.finish);

		pthread_mutex_lock(&mt.lock);
		for (w = mt.main.next; w != &mt.main; w = w->next)//终止所有任务线程
			pthread_cancel(w->thread_id);
		mt.exit = 1;
		pthread_mutex_unlock(&mt.lock);

		while (mt.main.next != &mt.main)
			fuse_join_worker(&mt, mt.main.next);//回收任务线程

		err = mt.error;
	}

	pthread_mutex_destroy(&mt.lock);
	sem_destroy(&mt.finish);
	if(se->error != 0)
		err = se->error;
	fuse_session_reset(se);

	if (created_config) {
		fuse_loop_cfg_destroy(config);
		config = NULL;
	}

	return err;
}

1.3 实际创建线程并执行event loop

static int fuse_loop_start_thread(struct fuse_mt *mt)
{
	int res;

	struct fuse_worker *w = malloc(sizeof(struct fuse_worker));
	if (!w) {
		fuse_log(FUSE_LOG_ERR, "fuse: failed to allocate worker structure\n");
		return -1;
	}
	memset(w, 0, sizeof(struct fuse_worker));
	w->fbuf.mem = NULL;
	w->mt = mt;

	w->ch = NULL;
	//检查是否需要为每个线程单独分配一个/dev/fuse的fd
	if (mt->clone_fd) {
		w->ch = fuse_clone_chan(mt);
		if(!w->ch) {
			/* Don't attempt this again */
			fuse_log(FUSE_LOG_ERR, "fuse: trying to continue "
				"without -o clone_fd.\n");
			mt->clone_fd = 0;
		}
	}

	res = fuse_start_thread(&w->thread_id, fuse_do_work, w);//调用该函数执行实际线程创建
	if (res == -1) {
		fuse_chan_put(w->ch);
		free(w);
		return -1;
	}
	list_add_worker(w, &mt->main);//添加到工作线程链表中
	mt->numavail ++;
	mt->numworker ++;

	return 0;
}

   在fuse_start_thread函数中会创建新线程，新线程执行fuse_do_work函数，这个函数才是实际执行event loop的函数。

static void *fuse_do_work(void *data)
{
	struct fuse_worker *w = (struct fuse_worker *) data;
	struct fuse_mt *mt = w->mt;

	while (!fuse_session_exited(mt->se)) {
		int isforget = 0;
		int res;

		pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
		res = fuse_session_receive_buf_int(mt->se, &w->fbuf, w->ch);//接收fuse内核传递的数据包，无数据读取时会阻塞住
		pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL);
		if (res == -EINTR)
			continue;
		if (res <= 0) {
			if (res < 0) {
				fuse_session_exit(mt->se);
				mt->error = res;
			}
			break;
		}

		pthread_mutex_lock(&mt->lock);
		if (mt->exit) {
			pthread_mutex_unlock(&mt->lock);
			return NULL;
		}

		/*
		 * This disgusting hack is needed so that zillions of threads
		 * are not created on a burst of FORGET messages
		 */
		if (!(w->fbuf.flags & FUSE_BUF_IS_FD)) {
			struct fuse_in_header *in = w->fbuf.mem;

			if (in->opcode == FUSE_FORGET ||
			    in->opcode == FUSE_BATCH_FORGET)
				isforget = 1;
		}

		if (!isforget)
			mt->numavail--;
		if (mt->numavail == 0 && mt->numworker < mt->max_threads)
			fuse_loop_start_thread(mt);//可用线程数小于最大线程数时创建新线程
		pthread_mutex_unlock(&mt->lock);
		//处理收到的数据包
		fuse_session_process_buf_int(mt->se, &w->fbuf, w->ch);

		pthread_mutex_lock(&mt->lock);
		if (!isforget)//处理完了，可用线程数+1
			mt->numavail++;

		/* creating and destroying threads is rather expensive - and there is
		 * not much gain from destroying existing threads. It is therefore
		 * discouraged to set max_idle to anything else than -1. If there
		 * is indeed a good reason to destruct threads it should be done
		 * delayed, a moving average might be useful for that.
		 */
		if (mt->max_idle != -1 && mt->numavail > mt->max_idle) {
			if (mt->exit) {
				pthread_mutex_unlock(&mt->lock);
				return NULL;
			}
			list_del_worker(w);
			mt->numavail--;
			mt->numworker--;
			pthread_mutex_unlock(&mt->lock);

			pthread_detach(w->thread_id);
			free(w->fbuf.mem);
			fuse_chan_put(w->ch);
			free(w);
			return NULL;
		}
		pthread_mutex_unlock(&mt->lock);
	}

	sem_post(&mt->finish);

	return NULL;
}

   其中接收buf主要通过res = read(ch ? ch->fd : se->fd, buf->mem, se->bufsize);，处理buf通过fuse_session_process_buf_int函数。处理buf的过程主要就是fuse数据协议的解析过程。该过程将会在后文讲解。