＜伍＞kbengine源码剖析之线程池

线程池的代码目录是在lib->thread中

1.变量定义

    bool isInitialize_;												// 线程池是否被初始化过
	//使用stl队列实现正在处理列表
	std::queue<TPTask*> bufferedTaskList_;							// 系统处于繁忙时还未处理的任务列表
    //使用stl双向链表实现
	std::list<TPTask*> finiTaskList_;								// 已经完成的任务列表
	size_t finiTaskList_count_;

	THREAD_MUTEX bufferedTaskList_mutex_;							// 处理bufferTaskList互斥锁
	THREAD_MUTEX threadStateList_mutex_;							// 处理bufferTaskList and freeThreadList_互斥锁
	THREAD_MUTEX finiTaskList_mutex_;								// 处理finiTaskList互斥锁
	//使用stl双向链表实现
	std::list<TPThread*> busyThreadList_;							// 繁忙的线程列表
	std::list<TPThread*> freeThreadList_;							// 闲置的线程列表
	std::list<TPThread*> allThreadList_;							// 所有的线程列表

	uint32 maxThreadCount_;											// 最大线程总数
	uint32 extraNewAddThreadCount_;									// 如果normalThreadCount_不足够使用则会新创建这么多线程
	uint32 currentThreadCount_;										// 当前线程数
	uint32 currentFreeThreadCount_;									// 当前闲置的线程数
	uint32 normalThreadCount_;										// 标准状态下的线程总数 即：默认情况下一启动服务器就开启这么多线程
																	// 如果线程不足够，则会新创建一些线程， 最大能够到maxThreadNum.

	bool isDestroyed_;

2.线程池初始化threadpool.cpp

THREAD_MUTEX_INIT(threadStateList_mutex_);	
THREAD_MUTEX_INIT(bufferedTaskList_mutex_);
THREAD_MUTEX_INIT(finiTaskList_mutex_);

以上代码是创建线程池，windows下使用的是初始化临界区

#define THREAD_MUTEX_INIT(x)								InitializeCriticalSection(&x)

linux下使用的是互斥锁

#define THREAD_MUTEX_INIT(x)								pthread_mutex_init (&x, NULL)

3.线程池的创建

    assert(!isInitialize_);
	INFO_MSG("ThreadPool::createThreadPool: creating  threadpool...\n");
	
	extraNewAddThreadCount_ = inewThreadCount;
	normalThreadCount_ = inormalMaxThreadCount;
	maxThreadCount_ = imaxThreadCount;
	
	for(uint32 i=0; i<normalThreadCount_; ++i)
	{
		TPThread* tptd = createThread(0);
		
		if(!tptd)
		{
			ERROR_MSG("ThreadPool::createThreadPool: error! \n");
			return false;
		}

		currentFreeThreadCount_++;	
		currentThreadCount_++;
		
		freeThreadList_.push_back(tptd);
		allThreadList_.push_back(tptd);
	}
	
	INFO_MSG(fmt::format("ThreadPool::createThreadPool: successfully({0}), "
		"newThreadCount={1}, normalMaxThreadCount={2}, maxThreadCount={3}\n",
			currentThreadCount_, extraNewAddThreadCount_, normalThreadCount_, maxThreadCount_));

	isInitialize_ = true;
	KBEngine::sleep(100);

参考以下配置文件

	<thread_pool>
		<!-- 默认超时时间(秒) 
			(default timeout(seconds))
		-->
		<timeout> 300.0 </timeout>
		
		<init_create> 1 </init_create>
		<pre_create> 2 </pre_create>
		<max_create> 8 </max_create>
	</thread_pool>

4.线程的创建

1>线程初始化

TPThread(ThreadPool* threadPool, int threadWaitSecond = 0):
	threadWaitSecond_(threadWaitSecond), 
	currTask_(NULL), 
	threadPool_(threadPool)
	{
		state_ = THREAD_STATE_SLEEP;
		initCond();
		initMutex();
	}

//windows下使用
#define THREAD_SINGNAL_INIT(x)								x = CreateEvent(NULL, TRUE, FALSE, NULL)
#define THREAD_MUTEX_INIT(x)								InitializeCriticalSection(&x)

//linux下使用
#define THREAD_SINGNAL_INIT(x)								pthread_cond_init(&x, NULL)
#define THREAD_MUTEX_INIT(x)								pthread_mutex_init (&x, NULL)

2>创建线程

//win32
#if KBE_PLATFORM == PLATFORM_WIN32
	tidp_ = (THREAD_ID)_beginthreadex(NULL, 0, 
		&TPThread::threadFunc, (void*)this, NULL, 0);
#else	
	if(pthread_create(&tidp_, NULL, TPThread::threadFunc, 
		(void*)this)!= 0)
	{
		ERROR_MSG("createThread error!");
	}
#endif
	return tidp_;

3>线程回调 __stdcall详解

//win32线程回调
unsigned __stdcall TPThread::threadFunc(void *arg)
{
    TPThread * tptd = static_cast<TPThread*>(arg);
	ThreadPool* pThreadPool = tptd->threadPool();

	bool isRun = true;
	tptd->reset_done_tasks();
    tptd->onStart();

	while(isRun)
	{
		if(tptd->task() != NULL)
		{
			isRun = true;
		}
		else
		{
			tptd->reset_done_tasks();
            
			isRun = tptd->onWaitCondSignal();
		}

		if(!isRun || pThreadPool->isDestroyed())
		{
			if(!pThreadPool->hasThread(tptd))
				tptd = NULL;

			goto __THREAD_END__;
		}

		TPTask * task = tptd->task();
		if(task == NULL)
			continue;

		tptd->state_ = THREAD_STATE_BUSY;

		while(task && !tptd->threadPool()->isDestroyed())
		{
			tptd->inc_done_tasks();
			tptd->onProcessTaskStart(task);
			tptd->processTask(task);
			tptd->onProcessTaskEnd(task);
			
			// 尝试继续从任务队列里取出一个繁忙的未处理的任务
			TPTask * task1 = tptd->tryGetTask();

			if(!task1)
			{
				tptd->state_ = THREAD_STATE_PENDING;
				tptd->onTaskCompleted();
				break;
			}
			else
			{
				pThreadPool->addFiniTask(task);
				task = task1;
				tptd->task(task1);
			}
		}
	}

__THREAD_END__:
	if(tptd)
	{
		TPTask * task = tptd->task();
		if(task)
		{
			WARNING_MSG(fmt::format("TPThread::threadFunc: task {0:p} not finish, thread.{1:p} will exit.\n", 
				(void*)task, (void*)tptd));

			delete task;
		}

		tptd->onEnd();
		tptd->state_ = THREAD_STATE_END;
		tptd->reset_done_tasks();
	}
	return 0;
}

void* TPThread::threadFunc(void* arg)
{
    TPThread * tptd = static_cast<TPThread*>(arg);
	ThreadPool* pThreadPool = tptd->threadPool();

	bool isRun = true;
	tptd->reset_done_tasks();

    pthread_detach(pthread_self());
    tptd->onStart();

	while(isRun)
	{
		if(tptd->task() != NULL)
		{
			isRun = true;
		}
		else
		{
			tptd->reset_done_tasks();
			isRun = tptd->onWaitCondSignal();
		}

		if(!isRun || pThreadPool->isDestroyed())
		{
			if(!pThreadPool->hasThread(tptd))
				tptd = NULL;

			goto __THREAD_END__;
		}

		TPTask * task = tptd->task();
		if(task == NULL)
			continue;

		tptd->state_ = THREAD_STATE_BUSY;

		while(task && !tptd->threadPool()->isDestroyed())
		{
			tptd->inc_done_tasks();
			tptd->onProcessTaskStart(task);
			tptd->processTask(task);
			tptd->onProcessTaskEnd(task);
			
			// 尝试继续从任务队列里取出一个繁忙的未处理的任务
			TPTask * task1 = tptd->tryGetTask();

			if(!task1)
			{
				tptd->state_ = THREAD_STATE_PENDING;
				tptd->onTaskCompleted();
				break;
			}
			else
			{
				pThreadPool->addFiniTask(task);
				task = task1;
				tptd->task(task1);
			}
		}
	}

__THREAD_END__:
	if(tptd)
	{
		TPTask * task = tptd->task();
		if(task)
		{
			WARNING_MSG(fmt::format("TPThread::threadFunc: task {0:p} not finish, thread.{1:p} will exit.\n", 
				(void*)task, (void*)tptd));

			delete task;
		}

		tptd->onEnd();
		tptd->state_ = THREAD_STATE_END;
		tptd->reset_done_tasks();
	}
    pthread_exit(NULL);
	return NULL;
}

4.线程池的添加与删除任务

//添加任务
bool ThreadPool::addTask(TPTask* tptask)
{
    THREAD_MUTEX_LOCK(threadStateList_mutex_);
	if(currentFreeThreadCount_ > 0)
	{
		bool ret = _addTask(tptask);
		THREAD_MUTEX_UNLOCK(threadStateList_mutex_);

		return ret;
	}
    ...
}
//
bool ThreadPool::_addTask(TPTask* tptask)
{
	std::list<TPThread*>::iterator itr = freeThreadList_.begin();
	TPThread* tptd = (TPThread*)(*itr);
	freeThreadList_.erase(itr);
	busyThreadList_.push_back(tptd);
	--currentFreeThreadCount_;

	//INFO_MSG("ThreadPool::currFree:%d, currThreadCount:%d, busy:[%d]\n",
	//		 currentFreeThreadCount_, currentThreadCount_, busyThreadList_count_);

	tptd->task(tptask);

#if KBE_PLATFORM == PLATFORM_WIN32
	if (tptd->sendCondSignal() == 0) {
#else
	if (tptd->sendCondSignal() != 0) {
#endif
		ERROR_MSG("ThreadPool::addTask: pthread_cond_signal error!\n");
		return false;
	}

	return true;
}

//线程回调
tptd->onTaskCompleted();
//完成后回调线程池，线程池进行线程回收
void TPThread::onTaskCompleted(void)
{
	threadPool_->addFiniTask(currTask_);
	currTask_ = NULL;
	threadPool_->addFreeThread(this);
}
//添加空闲线程
bool ThreadPool::addFreeThread(TPThread* tptd)
{
	THREAD_MUTEX_LOCK(threadStateList_mutex_);
	std::list<TPThread*>::iterator itr;
	itr = find(busyThreadList_.begin(), busyThreadList_.end(), tptd);

	if(itr != busyThreadList_.end())
	{
		busyThreadList_.erase(itr);
	}
	else
	{
		THREAD_MUTEX_UNLOCK(threadStateList_mutex_);

		ERROR_MSG(fmt::format("ThreadPool::addFreeThread: busyThreadList_ not found thread.{0}\n",
		 (uint32)tptd->id()));
		
		delete tptd;
		return false;
	}
		
	freeThreadList_.push_back(tptd);
	currentFreeThreadCount_++;
	THREAD_MUTEX_UNLOCK(threadStateList_mutex_);
	return true;
}

5.线程池销毁

void ThreadPool::destroy()
{
	isDestroyed_ = true;

	THREAD_MUTEX_LOCK(threadStateList_mutex_);

	DEBUG_MSG(fmt::format("ThreadPool::destroy(): starting size {0}.\n",
		allThreadList_.size()));
	
	THREAD_MUTEX_UNLOCK(threadStateList_mutex_);

	int itry = 0;
	while(true)
	{
		KBEngine::sleep(300);
		itry++;

		std::string taskaddrs = "";
		THREAD_MUTEX_LOCK(threadStateList_mutex_);

		int count = (int)allThreadList_.size();
		std::list<TPThread*>::iterator itr = allThreadList_.begin();
		for(; itr != allThreadList_.end(); ++itr)
		{
			if((*itr))
			{
				if((*itr)->state() != TPThread::THREAD_STATE_END)
				{
					(*itr)->sendCondSignal();
					taskaddrs += (fmt::format("{0:p},", (void*)(*itr)));
				}
				else
				{
					count--;
				}
			}
		}

		THREAD_MUTEX_UNLOCK(threadStateList_mutex_);
		
		if(count <= 0)
		{
			break;
		}
		else
		{
			WARNING_MSG(fmt::format("ThreadPool::destroy(): waiting for thread({0})[{1}], try={2}\n", 
				count, taskaddrs, itry));
		}
	}

	THREAD_MUTEX_LOCK(threadStateList_mutex_);

	KBEngine::sleep(100);

	std::list<TPThread*>::iterator itr = allThreadList_.begin();
	for(; itr != allThreadList_.end(); ++itr)
	{
		if((*itr))
		{
			delete (*itr);
			(*itr) = NULL;
		}
	}
	
	allThreadList_.clear();
	THREAD_MUTEX_UNLOCK(threadStateList_mutex_);

	THREAD_MUTEX_LOCK(finiTaskList_mutex_);
	
	if(finiTaskList_.size() > 0)
	{
		WARNING_MSG(fmt::format("ThreadPool::~ThreadPool(): Discarding {0} finished tasks.\n",
			finiTaskList_.size()));

		std::list<TPTask*>::iterator finiiter  = finiTaskList_.begin();
		for(; finiiter != finiTaskList_.end(); ++finiiter)
		{
			delete (*finiiter);
		}
	
		finiTaskList_.clear();
		finiTaskList_count_ = 0;
	}

	THREAD_MUTEX_UNLOCK(finiTaskList_mutex_);

	THREAD_MUTEX_LOCK(bufferedTaskList_mutex_);

	if(bufferedTaskList_.size() > 0)
	{
		WARNING_MSG(fmt::format("ThreadPool::~ThreadPool(): Discarding {0} buffered tasks.\n", 
			bufferedTaskList_.size()));

		while(bufferedTaskList_.size() > 0)
		{
			TPTask* tptask = bufferedTaskList_.front();
			bufferedTaskList_.pop();
			delete tptask;
		}
	}
	
	THREAD_MUTEX_UNLOCK(bufferedTaskList_mutex_);

	THREAD_MUTEX_DELETE(threadStateList_mutex_);
	THREAD_MUTEX_DELETE(bufferedTaskList_mutex_);
	THREAD_MUTEX_DELETE(finiTaskList_mutex_);

	DEBUG_MSG("ThreadPool::destroy(): successfully!\n");
}