Web服务器项目详解

一、新连接到来的处理流程

  一个新的连接到来后，首先被MainReactor接收，然后通过轮询调度的方式（避免了惊群效应）分配给某个subReactor，因为涉及到主线程和IO线程竞争，所以需要加锁。

  subReactor这个时候可能正阻塞在epoll_wait上，所以需要异步唤醒subReactor的IO线程去接收新连接，并关注该文件描述符上是否有事件发生。

  如果文件描述符上有事件发生，epoll_wait会返回活跃的文件描述符，然后回调之前被注册的事件函数。

二、Channel、TcpConnection、TcpServer、Poller、EventLoop类详解

1、Channel类

  Channel类表示每一个客户端连接的通道，封装了文件描述符并负责接收从TcpConnection类传过来的事件回调函数，以后当文件描述符上有事件发生时直接回调；每一个套接字对应于一个Channel。

#ifndef _CHANNEL_H_
#define _CHANNEL_H_

#include <functional>

class Channel
{
public:
    //回调函数类型
    typedef std::function<void()> Callback;

    Channel();
    ~Channel();

    //设置文件描述符
    void SetFd(int fd) 
    {
        fd_ = fd; 
    }

    //获取文件描述符
    int GetFd() const
    { 
        return fd_; 
    }    

    //设置触发事件
    void SetEvents(uint32_t events)
    { 
        events_ = events; 
    }

    //获取触发事件
    uint32_t GetEvents() const
    { 
        return events_; 
    }

    //事件分发处理
    void HandleEvent();

    //设置读事件回调
    void SetReadHandle(const Callback &cb)
    {
        readhandler_ = cb; //提高效率，可以使用move语义,这里暂时还是存在一次拷贝
    }

    //设置写事件回调
    void SetWriteHandle(const Callback &cb)
    {
        writehandler_ = cb; 
    }    

    //设置错误事件回调
    void SetErrorHandle(const Callback &cb)
    { 
        errorhandler_ = cb;
    }

    //设置close事件回调
    void SetCloseHandle(const Callback &cb)
    {
        closehandler_ = cb;
    }

private:
    //文件描述符
    int fd_;
    //事件，一般情况下为epoll events 
    uint32_t events_;

    //事件触发时执行的函数，在tcpconn中注册
    Callback readhandler_;
    Callback writehandler_;
    Callback errorhandler_;
    Callback closehandler_;
};

#endif

2、TcpConnection类

  TcpConnection类是对客户端连接（文件描述符和地址）的抽象，负责向channel类注册事件（可读、可写、错误等），以后如果有事件发生时还会负责数据的收发。

#ifndef _TCP_CONNECTION_H_
#define _TCP_CONNECTION_H_

#include <functional>
#include <string>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <thread>
#include <memory>
#include "Channel.h"
#include "EventLoop.h"

class TcpConnection : public std::enable_shared_from_this<TcpConnection>
{
public:
    //TcpConnection智能指针
    typedef std::shared_ptr<TcpConnection> spTcpConnection;

    //回调函数类型
    typedef std::function<void(const spTcpConnection&)> Callback;
    typedef std::function<void(const spTcpConnection&, std::string&)> MessageCallback;
    //typedef std::function<void()> TaskCallback;.

    TcpConnection(EventLoop *loop, int fd, const struct sockaddr_in &clientaddr);
    ~TcpConnection();

    //获取当前连接的fd
    int fd() const
    { return fd_; }

    //获取当前连接所属的loop
    EventLoop* GetLoop() const { return loop_; }

    //添加本连接对应的事件到loop
    void AddChannelToLoop();

    //发送数据的函数
    void Send(const std::string &s); 

    //在当前IO线程发送数据函数
    void SendInLoop(); 

    //主动清理连接
    void Shutdown(); 

    //在当前IO线程清理连接函数
    void ShutdownInLoop();

    //可读事件回调
    void HandleRead(); 
    //可写事件回调
    void HandleWrite(); 
    //错误事件回调
    void HandleError(); 
    //连接关闭事件回调
    void HandleClose(); 

    //设置收到数据回调函数
    void SetMessaeCallback(const MessageCallback &cb)
    {
        messagecallback_ = cb;
    }

    //设置发送完数据的回调函数
    void SetSendCompleteCallback(const Callback &cb)
    {
        sendcompletecallback_ = cb;
    }

    //设置连接关闭的回调函数
    void SetCloseCallback(const Callback &cb)
    {
        closecallback_ = cb;
    }

    //设置连接异常的回调函数
    void SetErrorCallback(const Callback &cb)
    {
        errorcallback_ = cb;
    }

    //设置连接清理函数
    void SetConnectionCleanUp(const Callback &cb)
    {
        connectioncleanup_ = cb;
    }

    //设置异步处理标志，开启工作线程池的时候使用
    void SetAsyncProcessing(const bool asyncprocessing)
    {
        asyncprocessing_ = asyncprocessing;
    }

private:
    //当前连接所在的loop
    EventLoop *loop_;

    //当前连接的事件
    std::unique_ptr<Channel> spchannel_;

    //文件描述符
    int fd_;

    //对端地址
    struct sockaddr_in clientaddr_;

    //半关闭标志位
    bool halfclose_; 

    //连接已关闭标志位
    bool disconnected_; 

    //异步调用标志位,当工作任务交给线程池时，置为true，任务完成回调时置为false
    bool asyncprocessing_; 

    //读写缓冲
    std::string bufferin_;
    std::string bufferout_;

    //各种回调函数
    MessageCallback messagecallback_;
    Callback sendcompletecallback_;
    Callback closecallback_;
    Callback errorcallback_;
    Callback connectioncleanup_;
};

#endif

3、TcpServer类

  TcpServer类负责处理新连接，并把新连接通过轮询调度的策略转发给subReactor，同时还要接收从用户传过来的事件处理函数，并创建一个TcpConnection类，把事件处理函数传递给TcpConnection类，最终这些函数都会传递到channel类上。

#include "TcpServer.h"
#include <iostream>
#include <stdio.h>
#include <unistd.h>
#include <fcntl.h>
#include <stdlib.h>
#include <sys/epoll.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <memory>

void Setnonblocking(int fd);

TcpServer::TcpServer(EventLoop* loop, const int port, const int threadnum)
    : serversocket_(),
    loop_(loop),
    serverchannel_(),
    conncount_(0),
    eventloopthreadpool(loop, threadnum)
{
    //serversocket_.SetSocketOption(); 
    serversocket_.SetReuseAddr();   
    serversocket_.BindAddress(port);
    serversocket_.Listen();
    serversocket_.Setnonblocking();

    serverchannel_.SetFd(serversocket_.fd());
    serverchannel_.SetReadHandle(std::bind(&TcpServer::OnNewConnection, this));
    serverchannel_.SetErrorHandle(std::bind(&TcpServer::OnConnectionError, this));
    
}

TcpServer::~TcpServer()
{

}

void TcpServer::Start()
{
    eventloopthreadpool.Start();

    serverchannel_.SetEvents(EPOLLIN | EPOLLET);
    loop_->AddChannelToPoller(&serverchannel_);
}

//新TCP连接处理，核心功能，业务功能注册，任务分发
void TcpServer::OnNewConnection()
{
    //循环调用accept，获取所有的建立好连接的客户端fd
    struct sockaddr_in clientaddr;
    int clientfd;
    while( (clientfd = serversocket_.Accept(clientaddr)) > 0) 
    {
        std::cout << "New client from IP:" << inet_ntoa(clientaddr.sin_addr) 
            << ":" << ntohs(clientaddr.sin_port) << std::endl;
        
        if(++conncount_ >= MAXCONNECTION)
        {
            close(clientfd);
            continue;
        }
        Setnonblocking(clientfd);

        //选择IO线程loop
        EventLoop *loop = eventloopthreadpool.GetNextLoop();

        //创建连接，注册业务函数
        std::shared_ptr<TcpConnection> sptcpconnection = std::make_shared<TcpConnection>(loop, clientfd, clientaddr);
        sptcpconnection->SetMessaeCallback(messagecallback_);
        sptcpconnection->SetSendCompleteCallback(sendcompletecallback_);
        sptcpconnection->SetCloseCallback(closecallback_);
        sptcpconnection->SetErrorCallback(errorcallback_);
        sptcpconnection->SetConnectionCleanUp(std::bind(&TcpServer::RemoveConnection, this, std::placeholders::_1));
        {
            std::lock_guard<std::mutex> lock(mutex_);
            tcpconnlist_[clientfd] = sptcpconnection;
        }
        

        newconnectioncallback_(sptcpconnection);
        //Bug，应该把事件添加的操作放到最后,否则bug segement fault,导致HandleMessage中的phttpsession==NULL
        //总之就是做好一切准备工作再添加事件到epoll！！！
        sptcpconnection->AddChannelToLoop();
    }
}

//连接清理,bugfix:这里应该由主loop来执行，投递回主线程删除 OR 多线程加锁删除
void TcpServer::RemoveConnection(std::shared_ptr<TcpConnection> sptcpconnection)
{
    std::lock_guard<std::mutex> lock(mutex_);
    --conncount_;
    //std::cout << "clean up connection, conncount is" << conncount_ << std::endl;   
    tcpconnlist_.erase(sptcpconnection->fd());
}

void TcpServer::OnConnectionError()
{    
    std::cout << "UNKNOWN EVENT" << std::endl;
    serversocket_.Close();
}

void Setnonblocking(int fd)
{
    int opts = fcntl(fd, F_GETFL);
    if (opts < 0)
    {
        perror("fcntl(fd,GETFL)");
        exit(1);
    }
    if (fcntl(fd, F_SETFL, opts | O_NONBLOCK) < 0)
    {
        perror("fcntl(fd,SETFL,opts)");
        exit(1);
    }
}

4、Poller类

  Poller类调用epoll_wait返回有事件发生的文件描述符，并更新channel上的事件，然后添加到活跃文件描述符列表上（activechannellist）。

#include "Poller.h"
#include <iostream>
#include <stdio.h> //perror
#include <stdlib.h> //exit
#include <unistd.h> //close
#include <errno.h>

#define EVENTNUM 4096 //最大触发事件数量
#define TIMEOUT 1000 //epoll_wait 超时时间设置

Poller::Poller(/* args */)
    : pollfd_(-1),
    eventlist_(EVENTNUM),
    channelmap_(),
    mutex_()
{
    pollfd_ = epoll_create(256);
    if(pollfd_ == -1)
    {
        perror("epoll_create1");
        exit(1);
    }
    std::cout << "epoll_create" << pollfd_ << std::endl;
}

Poller::~Poller()
{
    close(pollfd_);
}

//等待I/O事件
void Poller::poll(ChannelList &activechannellist)
{
    int timeout = TIMEOUT;
    //std::cout << "epoll_wait..." << std::endl;(int)eventlist_.capacity()
    int nfds = epoll_wait(pollfd_, &*eventlist_.begin(), (int)eventlist_.capacity(), timeout);
    //int nfds = epoll_wait(pollfd_, &*eventlist_.begin(), (int)channelmap_.size()*0.7+1, timeout);
    if(nfds == -1)
    {
        //printf("epoll_wait error code is:%d", errno);
        perror("epoll wait error");
        //exit(1);
    }
    //printf("event num:%d\n", nfds);
    //std::cout << "event num:" << nfds << "\n";// << std::endl;
    for(int i = 0; i < nfds; ++i)
    {
        int events = eventlist_[i].events;
        //int fd = eventlist_[i].data.fd;
        Channel *pchannel = (Channel*)eventlist_[i].data.ptr;
        int fd = pchannel->GetFd();

        std::map<int, Channel*>::const_iterator iter;
        {
            std::lock_guard <std::mutex> lock(mutex_);
            iter = channelmap_.find(fd);
        }        
        if(iter != channelmap_.end())
        {
            pchannel->SetEvents(events);
            activechannellist.push_back(pchannel);
        }
        else
        {            
            std::cout << "not find channel!" << std::endl;
        }
    }
    if(nfds == (int)eventlist_.capacity())
    {
        std::cout << "resize:" << nfds << std::endl;
        eventlist_.resize(nfds * 2);
    }
    //eventlist_.clear();

}

//添加事件
void Poller::AddChannel(Channel *pchannel)
{
    int fd = pchannel->GetFd();
    struct epoll_event ev;
    ev.events = pchannel->GetEvents();
    //data是联合体
    //ev.data.fd = fd;
    ev.data.ptr = pchannel;
    {
        std::lock_guard <std::mutex> lock(mutex_);
        channelmap_[fd] = pchannel;
    }      

    if(epoll_ctl(pollfd_, EPOLL_CTL_ADD, fd, &ev) == -1)
    {
        perror("epoll add error");
        exit(1);
    }
    //std::cout << "addchannel!" << std::endl;
}

//删除事件
void Poller::RemoveChannel(Channel *pchannel)
{
    int fd = pchannel->GetFd();
    struct epoll_event ev;
    ev.events = pchannel->GetEvents();
    ///ev.data.fd = fd
    ev.data.ptr = pchannel;
    {
        std::lock_guard <std::mutex> lock(mutex_);
        channelmap_.erase(fd);
    }    

    if(epoll_ctl(pollfd_, EPOLL_CTL_DEL, fd, &ev) == -1)
    {
        perror("epoll del error");
        exit(1);
    }
    //std::cout << "removechannel!" << std::endl;
}

//更新事件
void Poller::UpdateChannel(Channel *pchannel)
{
    int fd = pchannel->GetFd();
    struct epoll_event ev;
    ev.events = pchannel->GetEvents();
    //ev.data.fd = fd;
    ev.data.ptr = pchannel;

    if(epoll_ctl(pollfd_, EPOLL_CTL_MOD, fd, &ev) == -1)
    {
        perror("epoll update error");
        exit(1);
    }
    //std::cout << "updatechannel!" << std::endl;
}

5、EventLoop类

  EventLoop类根据Poller类返回的活跃文件描述符列表activechannellist，然后遍历该列表，依次调用channel上的回调函数。

#include "EventLoop.h"
#include <iostream>
#include <sys/eventfd.h>
#include <unistd.h>
#include <stdlib.h>

int CreateEventFd()
{
    int evtfd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
    if (evtfd < 0)
    {
        std::cout << "Failed in eventfd" << std::endl;
        exit(1);
    }
    return evtfd;
}

EventLoop::EventLoop(/* args */)
    : functorlist_(),
    channellist_(),
    activechannellist_(),
    poller_(),
    quit_(true),
    tid_(std::this_thread::get_id()),
    mutex_(),
    wakeupfd_(CreateEventFd()),
    wakeupchannel_()
{
    wakeupchannel_.SetFd(wakeupfd_);
    wakeupchannel_.SetEvents(EPOLLIN | EPOLLET);
    wakeupchannel_.SetReadHandle(std::bind(&EventLoop::HandleRead, this));
    wakeupchannel_.SetErrorHandle(std::bind(&EventLoop::HandleError, this));
    AddChannelToPoller(&wakeupchannel_);
}

EventLoop::~EventLoop()
{
    close(wakeupfd_);
}

void EventLoop::WakeUp()
{
    uint64_t one = 1;
    ssize_t n = write(wakeupfd_, (char*)(&one), sizeof one);
}

void EventLoop::HandleRead()
{
    uint64_t one = 1;
    ssize_t n = read(wakeupfd_, &one, sizeof one);
}

void EventLoop::HandleError()
{
    ;
}    

void EventLoop::loop()
{
    quit_ = false;
    while(!quit_)
    {
        poller_.poll(activechannellist_);
        for(Channel *pchannel : activechannellist_)
        {            
            pchannel->HandleEvent();//处理事件
        }
        activechannellist_.clear();
        ExecuteTask();
    }
}

三、这几个类之间的关系