libevent2.1.8保证线程安全的bufferevent写操作（读操作也一样）

编程环境：VS2013

libevent库：libevent2.1.8

本文贴出了主要的代码，看懂代码的前提是你之前已经对libevent如何使用有了基本的了解，如果没了解过的话可下载这里的代码学习下VS2013使用libevent2.1.8简单例子

下面将贴出多线程状态下使用libevent的主要代码

#ifndef _COMMANDER_SOCKET_LIBEVENT_H_
#define _COMMANDER_SOCKET_LIBEVENT_H_

#include <event2/bufferevent.h>  
#include <event2/buffer.h>  
#include <event2/listener.h>  
#include <event2/util.h>  
#include <event2/event.h>  
#include <event2/thread.h> 

#include <string>
#include <list>
using std::string;

#define MAX_BYTES    1024	//每次从缓冲区读取的最大字节数

typedef struct Socketfd_MsgSerial
{
	int _iSocketfd;//连接的客户端socketfd
	struct bufferevent *_bufferevent;//连接的客户端socketfd对应托管的bufferevent
	std::list<string> _lsSerial; //客户端socketfd需要发送报文中的序列号临时保存起来
}fd_serial;//将信息保存出来，方便其他地方进行写操作

class CCommanderSocket_libevent
{
public:
	CCommanderSocket_libevent();
	~CCommanderSocket_libevent();

private:
	int _localPort;
	bool _bStopServer;//true:服务线程结束 false:服务线程尚未结束
	std::list<fd_serial> _lsFd_Serial; //存储当前正在通讯的soketfd和报文中的序列号
	CRITICAL_SECTION _argsLock; //数据操作临界区

	/*!	\brief
	*	使用socket之前初始化WSAStartup
	*	\retval true:成功	false:失败
	*/
	bool _initWosa();
	/*!	\brief
	*	主线程，主线程内监听并接收客户端发送的报文进行处理
	*	\lpParam[in] 一般传入this自身对象指针
	*	\retval 0:成功 1:失败
	*/
	UINT _work(LPVOID lpParam);
	static UINT work_thread(LPVOID lpParam);

private:
	struct event_base *_base;
	static CCommanderSocket_libevent *cThis;

	/*!	\brief
	*	启动服务
	*	\iLocalPort[in] 用于本地TCP监听的端口
	*	\retval true:成功	false:失败
	*/
	bool _launchServices(const int iLocalPort);
	/*!	\brief
	*	bufferevent发生了输出流完成事件(即服务器缓冲区struct evbuffer *output数据被输出完成)时触发回调函数
	*	\bev[in] 发生了事件的bufferevent
	*	\user_data[in] 在bufferevent_setcb()时候设置的最后一个参数
	*	\retval null
	*/
	static void _write_callback(struct bufferevent *bev, void *user_data);
	/*!	\brief
	*	bufferevent发生了输入流完成事件(即服务器已经接收到数据放到struct evbuffer *input缓冲区)时触发回调函数
	*	\bev[in] 发生了事件的bufferevent
	*	\user_data[in] 在bufferevent_setcb()时候设置的最后一个参数
	*	\retval null
	*/
	static void _read_callback(struct bufferevent *bev, void *user_data);
	/*!	\brief
	*	当网络I/O出现错误，如链接中断，超时或其他错误时触发回调函数
	*	\bev[in] 发生了事件的bufferevent
	*	\what[in] 网络I/O错误标志
	*	\user_data[in] 在bufferevent_setcb()时候设置的最后一个参数
	*	\retval null
	*/
	static void _error_callback(struct bufferevent *bev, short what, void *user_data);
	/*!	\brief
	*	监听事件触发，触发机制参考evconnlistener_new_bind的事件类型及属性
	*	\listener[in] 用于监听的fd
	*	\what[in] 触发该回调的事件类型
	*	\user_data[in] 在event_new()时候设置的最后一个参数
	*	\retval null
	*/
	static void _listener_callback(struct evconnlistener *listener, evutil_socket_t fd, struct sockaddr *sa, int socklen, void *user_data);
};

#endif

#include "stdafx.h"
#include "CommanderSocket_libevent.h"
#include <sstream>

CCommanderSocket_libevent * CCommanderSocket_libevent::cThis = nullptr;

CCommanderSocket_libevent::CCommanderSocket_libevent()
{
	_localPort = 8999;
	_bStopServer = true;
	_lsFd_Serial.clear();
	::InitializeCriticalSection(&_argsLock);
	if (_initWosa())
	{
		cThis = this;
		AfxBeginThread(work_thread, this);
	}
}

CCommanderSocket_libevent::~CCommanderSocket_libevent()
{
	/*释放资源*/
	if (!_bStopServer)
	{
		std::list<fd_serial>::iterator iter = _lsFd_Serial.begin();
		while (iter != _lsFd_Serial.end())
		{
			bufferevent_free(iter->_bufferevent);//释放bufferevent同时会关闭底层传输端口
			++iter;
		}
		event_base_loopbreak(_base);
		while (!_bStopServer)
			_sleep(1000);
	}
	::DeleteCriticalSection(&_argsLock);
}

bool CCommanderSocket_libevent::_initWosa()
{
	WSADATA			wsaData;
	if (WSAStartup(MAKEWORD(2, 2), &wsaData))
	{
		WriteLog(LOG_LEVEL_FATAL, "WSAStartup UDP Error[%d].", WSAGetLastError());
		return false;
	}

	if (LOBYTE(wsaData.wVersion) != 2 || HIBYTE(wsaData.wVersion) != 2)
	{
		WriteLog(LOG_LEVEL_FATAL, "WSAStartup Version[%d] Error.", wsaData.wVersion);
		WSACleanup();
		return false;
	}
	return true;
}

UINT CCommanderSocket_libevent::work_thread(LPVOID lpParam)
{
	return ((CCommanderSocket_libevent*)lpParam)->_work(NULL);
}

UINT CCommanderSocket_libevent::_work(LPVOID lpParam)
{
	_bStopServer = false;
	WriteLog(LOG_LEVEL_INFO, "libevent socket service thread[_work] Start");
	cThis->_launchServices(_localPort);//成功的话会阻塞在函数里面
	WriteLog(LOG_LEVEL_INFO, "libevent socket service thread[_work] over");
	_bStopServer = true;
	return 0;
}

void CCommanderSocket_libevent::_write_callback(struct bufferevent *bev, void *user_data)
{
	int clientfd = (int)user_data;
	struct evbuffer *output = bufferevent_get_output(bev);
	if (evbuffer_get_length(output) == 0)
	{
		cThis->WriteLog(LOG_LEVEL_INFO, "client[fd=%d] response finish", clientfd);
	}
	else
	{
		cThis->WriteLog(LOG_LEVEL_INFO, "client[fd=%d] waiting response data len: %d", clientfd, evbuffer_get_length(output));
	}
}

void CCommanderSocket_libevent::_read_callback(struct bufferevent *bev, void *user_data)
{
	int clientfd = (int)user_data;
	int iPos, iMsgLen;
	struct evbuffer *input;
	char szData[MAX_BYTES + 1];
	string strRevMsg = "", strMsg;
	input = bufferevent_get_input(bev);//其实就是取出bufferevent中的input   

	size_t totalread_len = 0, curread_len = 0;
	size_t input_len = evbuffer_get_length(input);
	
	while (1)
	{
		memset(szData, '\0', sizeof(szData));
		curread_len = evbuffer_remove(input, szData, MAX_BYTES);//从evbuffer读取数据到data，成功返回读取到的字节数，失败返回-1
		if (-1 == curread_len)
		{
			cThis->WriteLog(LOG_LEVEL_WARN, "client[fd=%d] can not drain the buffer.", clientfd);
			break;
		}
		else
		{
			totalread_len += curread_len;
			strRevMsg += szData;
			if (totalread_len < input_len)//数据未读完  
			{
				cThis->WriteLog(LOG_LEVEL_INFO, "client[fd=%d] total len[%d] and current get buffer len[%d]", clientfd, totalread_len, curread_len);
				continue;
			}
			else
			{
				cThis->WriteLog(LOG_LEVEL_INFO, "client[fd=%d] get buffer finish: %s", clientfd, strRevMsg.c_str());
				/*分析接收的数据有没有粘包的情况*/
				iPos = strRevMsg.find("|");
				while (iPos != string::npos)
				{
					iMsgLen = atoi(strRevMsg.substr(0, iPos).c_str());//报文长度
					strMsg = strRevMsg.substr(iPos + 1);
					if (iMsgLen >= strMsg.length())
					{
						cThis->_addCmd(strMsg, clientfd);//这个接口是我处理报文的接口，就不贴出来了
						break;
					}
					else
					{
						strRevMsg = strMsg.substr(iMsgLen);
						strMsg = strMsg.substr(0, iMsgLen);
						cThis->_addCmd(strMsg, clientfd);
						iPos = strRevMsg.find("|");
					}
				}
				if (iPos == string::npos)
					cThis->_addCmd(strRevMsg, clientfd);
				break;
			}
		}
	}
}

void CCommanderSocket_libevent::_error_callback(struct bufferevent *bev, short what, void *user_data)
{
	int clientfd = (int)user_data;
	if (what & BEV_EVENT_EOF)//遇到文件结束指示,即与客户端的连接中断
	{
		cThis->WriteLog(LOG_LEVEL_WARN, "client[fd=%d] connection closed", clientfd);
	}
	else if (what & BEV_EVENT_ERROR)//操作时发生了错误，EVUTIL_SOCKET_ERROR()查看本线程最后一次套接字操作的全局错误号
	{
		cThis->WriteLog(LOG_LEVEL_WARN, "client[fd=%d] get some other error %ld", clientfd, EVUTIL_SOCKET_ERROR());
	}
	else if (what & BEV_EVENT_TIMEOUT)//超时
	{
		cThis->WriteLog(LOG_LEVEL_WARN, "client[fd=%d] timed out and close the connection", clientfd);
	}
	bufferevent_free(bev);//释放bufferevent同时会关闭底层传输端口
	/*删除存储的客户端相关信息*/
	::EnterCriticalSection(&cThis->_argsLock);
	std::list<fd_serial>::iterator iterFD = cThis->_lsFd_Serial.begin();
	while (iterFD != cThis->_lsFd_Serial.end())
	{
		if ((*iterFD)._iSocketfd == clientfd)
		{
			cThis->_lsFd_Serial.erase(iterFD);
			break;
		}
		++iterFD;
	}
	::LeaveCriticalSection(&cThis->_argsLock);
}

void CCommanderSocket_libevent::_listener_callback(struct evconnlistener *listener, evutil_socket_t fd, struct sockaddr *sa, int socklen, void *user_data)
{
	struct event_base *base = (struct event_base *)user_data;
	struct bufferevent *bev;

	evutil_make_socket_nonblocking(fd);//设置非阻塞socket
	//BEV_OPT_CLOSE_ON_FREE表示释放bufferevent时关闭底层传输端口。这将关闭底层套接字，释放底层bufferevent等。 BEV_OPT_THREADSAFE：自动为 bufferevent 分配锁
	bev = bufferevent_socket_new(base, fd, BEV_OPT_CLOSE_ON_FREE | BEV_OPT_THREADSAFE);
	if (!bev) {
		cThis->WriteLog(LOG_LEVEL_ERROR, "Error constructing bufferevent");
		event_base_loopbreak(base);
		return;
	}

	// 设置超时时间，客户端连接上服务器，服务端在规定是否没有读到则调用conn_eventcb进行超时处理  
	/*struct timeval tv_read = { 10, 0 }, tv_write = { 10, 0 };
	bufferevent_set_timeouts(bev, &tv_read, &tv_write);*/

	/* BEV_OPT_THREADSAFE-支持 buffevent 多线程 */
	int ret = bufferevent_enable(bev, BEV_OPT_THREADSAFE);
	if (ret < 0)
		cThis->WriteLog(LOG_LEVEL_ERROR, "Failed to bufferevent_enable BEV_OPT_THREADSAFE");

	//设置回调函数,当有数据被读入完成的时候，readcb被调用；当服务器发送数据被输出完成的时候，writecb被调用；当网络I/O出现错误，如链接中断，超时或其他错误时，errorcb被调用；最后一个参数为传递给回调函数的参数
	bufferevent_setcb(bev, _read_callback, _write_callback, _error_callback, (void *)fd);
	//bufferevent_setwatermark(bev, EV_READ, 0, MAX_LINE);  
	//启用读写事件,其实是调用了event_add将相应读写事件加入事件监听队列poll。正如文档所说，如果相应事件不置为true，bufferevent是不会读写数据的
	//bufferevent_enable(bev, EV_READ | EV_WRITE);

	bufferevent_disable(bev, EV_WRITE);
	bufferevent_enable(bev, EV_READ);

	/*存储客户端连接信息*/
	fd_serial struInfo;
	struInfo._iSocketfd = fd;
	struInfo._bufferevent = bev;
	struInfo._lsSerial.clear();
	::EnterCriticalSection(&cThis->_argsLock);
	cThis->_lsFd_Serial.push_back(struInfo);
	::LeaveCriticalSection(&cThis->_argsLock);

	/*多线程操作evbuffer的时候就需要使用锁来保证原子性，读写操作都有相对应的锁*/
	//struct evbuffer *input = bufferevent_get_input(bev);
	//evbuffer_lock(input);//加锁
	//evbuffer_unlock(input);//解锁
	struct evbuffer *output = bufferevent_get_output(bev);
	evbuffer_lock(output);//加锁
	bufferevent_enable(bev, EV_WRITE);
	char *response = "welcome to connect server!";
	evbuffer_add(output, response, strlen(response));//添加需要发送的消息到evbuffer缓冲区
	evbuffer_unlock(output);//解锁
}

bool CCommanderSocket_libevent::_launchServices(const int iLocalPort)
{
	struct sockaddr_in sin;
	struct evconnlistener *listener;

	//支持多线程，需要线程安全,该函数一定要在event_base_new之前调用，如果在event_base_new之后才调用的话，那么该event_base就不会是线程安全的了
	evthread_use_windows_threads();

	_base = event_base_new();
	if (!_base)
	{
		cThis->WriteLog(LOG_LEVEL_WARN, "Failed to create event base");
		return false;
	}

	sin.sin_family = AF_INET;
	sin.sin_addr.s_addr = INADDR_ANY;//在本机的所有ip上开始监听,因为有些机子绑定了不同的ip
	sin.sin_port = htons(iLocalPort);

	/* LEV_OPT_THREADSAFE 支持 listener 多线程 */
	listener = evconnlistener_new_bind(_base, _listener_callback, (void *)_base,
		LEV_OPT_REUSEABLE | LEV_OPT_CLOSE_ON_FREE | LEV_OPT_THREADSAFE, -1,
		(struct sockaddr*)&sin,
		sizeof(sin));

	if (!listener) {
		cThis->WriteLog(LOG_LEVEL_WARN, "Could not create a listener!\n");
		return false;
	}
	event_base_dispatch(_base);//启动事件循环,阻塞在这里,要结束时间循环的话需要调用event_base_loopexit或者event_base_loopbreak
	evconnlistener_free(listener);
	event_base_free(_base);
	return true;
}

好了，接下来如果你在其他地方调用写操作的话，根据之前保存的bufferevent信息_lsFd_Serial找到对应的bufferevent对象bev就可以进行写操作了，对应的写操作：

struct evbuffer *output = bufferevent_get_output(bev);
	evbuffer_lock(output);//加锁
	bufferevent_enable(bev, EV_WRITE);
	char *response = "welcome to connect server!";
	evbuffer_add(output, response, strlen(response));//添加需要发送的消息到evbuffer缓冲区
	evbuffer_unlock(output);//解锁