libevent 多线程IO

前言：

        前阵子有朋友问起，如何进行linux开发，我一时也没法回答，我对linux也不是非常的了解，长时间从事android的开发，但是linux时常会接触到，勉强跟朋友交流了下。linux开发是个很麻烦的事情，相比Android的开发要好多了，至少我有这种感觉吧，Android的IDE比较好用。然而linux开发怎么入手，我觉得不管linux有多少需要我们去研究的，但至少经典的IO模型是需要我们去研究的，多路复用必然免不了。但是这些接口用起来也不方便，好在有前人为我们考虑了这样的问题。有非常多的库可以帮我们提升开发效率，libev，libevent，libuv等。

    摸索了一阵libevent，多线程IO的实例并不多，有个别实例，但是感觉用着不舒服，于是自己写了个。如下代码：

以下代码用作监听客户端的链接：

#ifndef CLIENTSOCKETACCEPTOR_H
#define CLIENTSOCKETACCEPTOR_H

#pragma once

#include <memory>
#include <vector>
#include <string>
#include <algorithm>
#include <thread>

#include <stdio.h>
#include <stdlib.h>
#include <netinet/in.h>
#include <unistd.h>
#include <map>

#include <event2/event.h>
#include <event2/listener.h>
#include "utils/events.h"

#include "ThreadPool.h"
#include "utils/events.h"
#include "utils/wrapper.h"

using namespace std;

class ClientSocketAcceptor {
private:
	/*
	 * 每个客户端对应一个fd，并有一个数据处理任务std::function<void()>
	 */
	std::map<evutil_socket_t,std::function<void()>> taskMap;

	/*
	 * 每个客户端对应一个event_base
	 */
	std::map<evutil_socket_t,raii_event_base> baseMap;

	/*
	 * 每个客户端对应一个event
	 */
	std::map<evutil_socket_t,raii_event> eventMap;

	std::map<evutil_socket_t,std::thread> threadMap;
	std::mutex syncMutex;

public:
	ClientSocketAcceptor();
	virtual ~ClientSocketAcceptor();

	static void connectionListener(struct evconnlistener *listener, evutil_socket_t fd, struct sockaddr *sa, int socklen, void *ctx);
	/*
	 * 尚未使用bufferevent
	 */
//	static void bufferevent_writecb(struct bufferevent *bev, void *ctx);
//	static void bufferevent_eventcb(struct bufferevent *bev, short events, void *ctx);
//	static void bufferevent_onRead(struct bufferevent *bev, void *ctx);

	static void onRead(evutil_socket_t socket_fd, short events, void *ctx);

//	void stop();
//	virtual void onDestroy();
};
#endif

类的实现：

#include <memory>
#include <vector>
#include <string>
#include <algorithm>
#include <thread>

#include <stdio.h>
#include <stdlib.h>
#include <netinet/in.h>
#include <unistd.h>
#include <string.h>

#include <event2/event.h>
#include <event2/listener.h>
#include "utils/events.h"

#include "ThreadPool.h"
#include "utils/events.h"
#include "utils/wrapper.h"
#include "ClientSocketAcceptor.h"

using namespace std;

#define BUF_SIZE 1024

ClientSocketAcceptor::ClientSocketAcceptor(){}
ClientSocketAcceptor::~ClientSocketAcceptor(){
	cout << " Exit the ClientSocketAcceptor" << endl;
	std::unique_lock<std::mutex> lock(syncMutex);
	for(auto i=baseMap.begin();i != baseMap.end(); i++){
//		event_base_loopexit(i->second.get(),nullptr);
		event_base_loopbreak(i->second.get());
	}
}

void ClientSocketAcceptor::connectionListener(struct evconnlistener *listener, evutil_socket_t fd, struct sockaddr *sa, int socklen, void *ctx){
	if (ctx == nullptr)
		return;

	ClientSocketAcceptor *acceptor = (ClientSocketAcceptor *)ctx;
	std::unique_lock<std::mutex> lock(acceptor->syncMutex);

	auto raii_base = obtain_event_base();
	auto base = raii_base.get();

	auto raii_socket_event = obtain_event(base, -1, 0, nullptr, nullptr);
	auto event = raii_socket_event.get();

	event_assign(event, base, fd, EV_READ|EV_PERSIST, ClientSocketAcceptor::onRead,acceptor);

	if (!event || event_add(event, nullptr) < 0) {
		cout << "Could not create/add a socket event!" << endl;
		close(fd);
		return;
	}
	acceptor->baseMap.emplace(make_pair(fd, std::move(raii_base)));
	acceptor->eventMap.emplace(make_pair(fd, std::move(raii_socket_event)));
	acceptor->taskMap.emplace(make_pair(fd, [](){
		// TODO: ADD YOUR CODE
		// 数据处理任务尚未实现
	}));

	/*
	 * 起一个新线程运行一个event io loop
	 */
	std::thread loop([acceptor](evutil_socket_t fd){
		// TODO: ADD YOUR CODE
		if(acceptor->baseMap.find(fd) != acceptor->baseMap.end()){
			cout << "Thread in the loop fd: " << fd << endl;
			event_base_dispatch(acceptor->baseMap.find(fd)->second.get());

			/*
			 * event looper 退出，将map清空。
			 */
			std::unique_lock<std::mutex> lock(acceptor->syncMutex);
			acceptor->baseMap.erase(fd);
			acceptor->eventMap.erase(fd);
			acceptor->taskMap.erase(fd);
			close(fd);
		}

		cout << "Thread exit fd: " << fd << endl;
	}, fd);

	loop.detach();
}
/*
 * 读取客户端数据，并返回给客户端（echo实现）
 */
void ClientSocketAcceptor::onRead(evutil_socket_t socket_fd, short events, void *ctx)
{
	if(ctx == nullptr)
		return;

	char buffer[BUF_SIZE];

	memset(buffer, 0 , BUF_SIZE);

	/*
	 * read the client data
	 */
	int size = TEMP_FAILURE_RETRY(read(socket_fd, (void *)buffer, BUF_SIZE));
    if(0 == size || -1 == size){//说明socket关闭
        cout<<"remote socket is close read size is " << size << " for socket: "<< socket_fd <<endl;
        auto acceptor = (ClientSocketAcceptor*)ctx;
        std::unique_lock<std::mutex> lock(acceptor->syncMutex);
		event_base_loopexit(acceptor->baseMap.find(socket_fd)->second.get(),
				nullptr);
        return;
    }
    cout << __func__ << " from fd:" << socket_fd<< " context is: " << buffer  << endl;

    /*
     * respone to the client
     */
    int ret = TEMP_FAILURE_RETRY(write(socket_fd, (void *)buffer, size));
    if (ret <= 0)
        cout << "Send response failed" << endl;
    else
    	cout << "Send response success ret: " << ret << endl;
}

使用如下：

#include <iostream>
#include <memory>
#include <stdio.h>
#include <event2/event-config.h>
#include <event2/event.h>
#include <event2/event_struct.h>
#include <event2/util.h>
#include <event2/listener.h>
#include <event2/bufferevent.h>
#include <event2/buffer.h>
#include <errno.h>
#include <string.h>

#include "utils/events.h"
#include "Handler.h"
#include "Message.h"

using namespace std;


#if 1
#include "ClientSocketAcceptor.h"

typedef void (*f_signal)(evutil_socket_t, short, void *);
static void
signal_cb(evutil_socket_t sig, short events, void *ctx)
{
	struct event_base *base = (struct event_base *)ctx;
	struct timeval delay = { 2, 0 };

	cout << "Caught an interrupt signal; exiting cleanly in two seconds. sig:" << sig << endl;

	event_base_loopexit(base, &delay);
}

int main(int argc, char **argv)
{
	auto flags = EV_SIGNAL|EV_PERSIST;
	struct sockaddr_in sin;

	memset(&sin, 0, sizeof(sin));
	sin.sin_family = AF_INET;
	sin.sin_port = htons(9950);

    // Obtain event base
    auto raii_base = obtain_event_base();

    std::unique_ptr<ClientSocketAcceptor> ptr_acceptor = make_unique<ClientSocketAcceptor>();

    /*
     * 监听客户端socket链接，
     * 通过回调connectionListener将客户端的socket fd存储起来
     */
	auto raii_listener = obtain_evconnlistener(raii_base.get(), ClientSocketAcceptor::connectionListener, (void*)ptr_acceptor.get(),
		    LEV_OPT_REUSEABLE|LEV_OPT_CLOSE_ON_FREE, -1, (struct sockaddr*)&sin, sizeof(sin));

	f_signal f = signal_cb;
    auto raii_signal_event = obtain_event(raii_base.get(), SIGINT, flags, f, (void*)raii_base.get());
    if(!raii_signal_event.get() || event_add(raii_signal_event.get(),nullptr) < 0){
		cout << "Could not create/add a signal event!" << endl;
		return 1;
    }

    /*
     * 起已个线程 监听客户端连接
     */
    std::thread acceptorThread([&raii_base](){

    	event_base_dispatch(raii_base.get());
    	cout << "acceptorThread exit" << endl;
    });

    acceptorThread.join();

//    ptr_acceptor = nullptr; //提前释放acceptor raii
    ptr_acceptor.reset(); //提前释放acceptor raii
    while(true)
    	std::this_thread::sleep_for(std::chrono::seconds(1000));

    cout << "main thread exit" << endl;
    return 0;
}

这里只帖出了部分代码的实现，详细请参考我的github

https://github.com/feekia/EventServer