QUIC项目epoll服务器分析

在前面的分析的过程中，
我们发现epoll服务器已经注册了两个事件：
这是proxy_server_bin.cc的代码

１、eps_->RegisterFD(fd, this, kEpollFlags);
２、 epoll_server_.RegisterFD(fd_, this, kEpollFlags);

注册完毕之后，就剩下等待事件发生

eps.WaitForEventsAndExecuteCallbacks();

查看这个函数：

void SimpleEpollServer::WaitForEventsAndExecuteCallbacks() {
  if (in_wait_for_events_and_execute_callbacks_) {
    EPOLL_LOG(DFATAL) << "Attempting to call WaitForEventsAndExecuteCallbacks"
                         " when an ancestor to the current function is already"
                         " WaitForEventsAndExecuteCallbacks!";
    // The line below is actually tested, but in coverage mode,
    // we never see it.
    return;  // COV_NF_LINE
  }
  AutoReset<bool> recursion_guard(&in_wait_for_events_and_execute_callbacks_,
                                  true);
  if (alarm_map_.empty()) {
    // no alarms, this is business as usual.
    WaitForEventsAndCallHandleEvents(timeout_in_us_, events_, events_size_);
    recorded_now_in_us_ = 0;
    return;
  }

  // store the 'now'. If we recomputed 'now' every iteration
  // down below, then we might never exit that loop-- any
  // long-running alarms might install other long-running
  // alarms, etc. By storing it here now, we ensure that
  // a more reasonable amount of work is done here.
  int64_t now_in_us = NowInUsec();

  // Get the first timeout from the alarm_map where it is
  // stored in absolute time.
  int64_t next_alarm_time_in_us = alarm_map_.begin()->first;
  EPOLL_VLOG(4) << "next_alarm_time = " << next_alarm_time_in_us
                << " now             = " << now_in_us
                << " timeout_in_us = " << timeout_in_us_;

  int64_t wait_time_in_us;
  int64_t alarm_timeout_in_us = next_alarm_time_in_us - now_in_us;

  // If the next alarm is sooner than the default timeout, or if there is no
  // timeout (timeout_in_us_ == -1), wake up when the alarm should fire.
  // Otherwise use the default timeout.
  if (alarm_timeout_in_us < timeout_in_us_ || timeout_in_us_ < 0) {
    wait_time_in_us = std::max(alarm_timeout_in_us, static_cast<int64_t>(0));
  } else {
    wait_time_in_us = timeout_in_us_;
  }

  EPOLL_VLOG(4) << "wait_time_in_us = " << wait_time_in_us;

  // wait for events.
  //关于时钟的调整不太明白，所以这里就只关注这里的等待事件的函数
  WaitForEventsAndCallHandleEvents(wait_time_in_us, events_, events_size_);
  CallAndReregisterAlarmEvents();
  recorded_now_in_us_ = 0;
}
//
void SimpleEpollServer::WaitForEventsAndCallHandleEvents(
    int64_t timeout_in_us, struct epoll_event events[], int events_size) {
  if (timeout_in_us == 0 || ready_list_.lh_first != NULL) {
    // If ready list is not empty, then don't sleep at all.
    timeout_in_us = 0;
  } else if (timeout_in_us < 0) {
    EPOLL_LOG(INFO) << "Negative epoll timeout: " << timeout_in_us
                    << "us; epoll will wait forever for events.";
    // If timeout_in_us is < 0 we are supposed to Wait forever.  This means we
    // should set timeout_in_us to -1000 so we will
    // Wait(-1000/1000) == Wait(-1) == Wait forever.
    timeout_in_us = -1000;
  } else {
    // If timeout is specified, and the ready list is empty.
    if (timeout_in_us < 1000) {
      timeout_in_us = 1000;
    }
  }
  const int timeout_in_ms = timeout_in_us / 1000;
  int64_t expected_wakeup_us = NowInUsec() + timeout_in_us;

  int nfds = epoll_wait_impl(epoll_fd_, events, events_size, timeout_in_ms);
  EPOLL_VLOG(3) << "nfds=" << nfds;

#ifdef EPOLL_SERVER_EVENT_TRACING
  event_recorder_.RecordEpollWaitEvent(timeout_in_ms, nfds);
#endif

  // If you're wondering why the NowInUsec() is recorded here, the answer is
  // simple: If we did it before the epoll_wait_impl, then the max error for
  // the ApproximateNowInUs() call would be as large as the maximum length of
  // epoll_wait, which can be arbitrarily long. Since this would make
  // ApproximateNowInUs() worthless, we instead record the time -after- we've
  // done epoll_wait, which guarantees that the maximum error is the amount of
  // time it takes to process all the events generated by epoll_wait.
  recorded_now_in_us_ = NowInUsec();

  if (timeout_in_us > 0) {
    int64_t delta = NowInUsec() - expected_wakeup_us;
    last_delay_in_usec_ = delta > 0 ? delta : 0;
  } else {
    // timeout_in_us < 0 means we waited forever until an event;
    // timeout_in_us == 0 means there was no kernel delay to track.
    last_delay_in_usec_ = 0;
  }

  if (nfds > 0) {
    for (int i = 0; i < nfds; ++i) {
      int event_mask = events[i].events;
      int fd = events[i].data.fd;
      //处理服务器上监听到的事件
      HandleEvent(fd, event_mask);
    }
/*

void SimpleEpollServer::HandleEvent(int fd, int event_mask) {
#ifdef EPOLL_SERVER_EVENT_TRACING
  event_recorder_.RecordEpollEvent(fd, event_mask);
#endif
  auto fd_i = cb_map_.find(CBAndEventMask(NULL, 0, fd));
  if (fd_i == cb_map_.end() || fd_i->cb == NULL) {
    // Ignore the event.
    // This could occur if epoll() returns a set of events, and
    // while processing event A (earlier) we removed the callback
    // for event B (and are now processing event B).
    return;
  }
  fd_i->events_asserted = event_mask;
  CBAndEventMask* cb_and_mask = const_cast<CBAndEventMask*>(&*fd_i);
  AddToReadyList(cb_and_mask);//将事件添加到readylist链中
}
*/
  } else if (nfds < 0) {
    // Catch interrupted syscall and just ignore it and move on.
    if (errno != EINTR && errno != 0) {
      int saved_errno = errno;
      char buf[kErrorBufferSize];
      EPOLL_LOG(FATAL) << "Error " << saved_errno << " in epoll_wait: "
                       << strerror_r(saved_errno, buf, sizeof(buf));
    }
  }

  // Now run through the ready list.
  if (ready_list_.lh_first) {
    CallReadyListCallbacks();//如果readylist链不为空，那么就开始执行回调函数
  }
}

查看这个回调函数：

void SimpleEpollServer::CallReadyListCallbacks() {
  // Check pre-conditions.
  DCHECK(tmp_list_.lh_first == NULL);
  // Swap out the ready_list_ into the tmp_list_ before traversing the list to
  //交换两条链
  // enable SetFDReady() to just push new items into the ready_list_.
  std::swap(ready_list_.lh_first, tmp_list_.lh_first);
  if (tmp_list_.lh_first) {
    tmp_list_.lh_first->entry.le_prev = &tmp_list_.lh_first;
    EpollEvent event(0);
    while (tmp_list_.lh_first != NULL) {
      DCHECK_GT(ready_list_size_, 0);
      CBAndEventMask* cb_and_mask = tmp_list_.lh_first;
      RemoveFromReadyList(*cb_and_mask);//遍历一个，那么就移除一个

      event.out_ready_mask = 0;
      event.in_events =
          cb_and_mask->events_asserted | cb_and_mask->events_to_fake;
      // TODO(fenix): get rid of the two separate fields in cb_and_mask.
      cb_and_mask->events_asserted = 0;
      cb_and_mask->events_to_fake = 0;
      {
        // OnEvent() may call UnRegister, so we set in_use, here. Any
        // UnRegister call will now simply set the cb to NULL instead of
        // invalidating the cb_and_mask object (by deleting the object in the
        // map to which cb_and_mask refers)
        AutoReset<bool> in_use_guard(&(cb_and_mask->in_use), true);
        //这里才是开始真正调用回调函数，每个注册的事件都有不同的OnEvent策略
        //接下来分析每个fd对应的事件
        cb_and_mask->cb->OnEvent(cb_and_mask->fd, &event);
      }

      // Since OnEvent may have called UnregisterFD, we must check here that
      // the callback is still valid. If it isn't, then UnregisterFD *was*
      // called, and we should now get rid of the object.
      if (cb_and_mask->cb == NULL) {
        cb_map_.erase(*cb_and_mask);
      } else if (event.out_ready_mask != 0) {
        cb_and_mask->events_to_fake = event.out_ready_mask;
        AddToReadyList(cb_and_mask);
      }
    }
  }
  DCHECK(tmp_list_.lh_first == NULL);
}

分析TunServer的OnEvent事件：

void OnEvent(int fd, QuicEpollEvent* event) override;
void TunServer::OnEvent(int fd, QuicEpollEvent* event){
    if(event->in_events & EPOLLIN){//先判断发生的是否是读事件
        ReadFromTun();//从tun中读取数据
    }
}
void TunServer::ReadFromTun(){
    char buffer[kBufLen];
    while(true){
    //从tun_fd中读取数据
        int ret=read((int)tuntap_get_fd(tun_device_),buffer,kBufLen);        
        if(ret<=0){//判断是是否是读取错误
            if(errno == EWOULDBLOCK || errno == EAGAIN){
                //read until no dada
            }else{
                QUIC_LOG(ERROR)<<"Tun Read Error";
            }
            break;
        }else{   
            //读取的字节要大于20       
            if((ret>=20)&&quic_){//读到数据了　从quic通道中讲数据发送出去
               quic_->SendToTunnel(buffer,ret);//这个留作稍后分析
            }
        }
    }
}

//下面查看QuicServer的fd

void MyQuicServer::OnEvent(int fd, QuicEpollEvent* event) {
  DCHECK_EQ(fd, fd_);
  event->out_ready_mask = 0;
//如果是读事件
  if (event->in_events & EPOLLIN) {
    QUIC_DVLOG(1) << "EPOLLIN";
       //读取数据　一直可以追踪到connect层
    dispatcher_->ProcessBufferedChlos(kNumSessionsToCreatePerSocketEvent);

    bool more_to_read = true;
    while (more_to_read) {
    //从dispatch中读取数据　　QuicPacketReader类中有两个buffer
      more_to_read = packet_reader_->ReadAndDispatchPackets(
          fd_, port_, QuicEpollClock(&epoll_server_), dispatcher_.get(),
          overflow_supported_ ? &packets_dropped_ : nullptr);
    }

    if (dispatcher_->HasChlosBuffered()) {
      // Register EPOLLIN event to consume buffered CHLO(s).
      event->out_ready_mask |= EPOLLIN;
    }
  }
  if (event->in_events & EPOLLOUT) {
  //这是写数据
    dispatcher_->OnCanWrite();
    if (dispatcher_->HasPendingWrites()) {
      event->out_ready_mask |= EPOLLOUT;
    }
  }
}

接着void MyQuicServer::OnEvent(int fd, QuicEpollEvent* event) 往下走，

//接受帧，并将数据分发给session
std::unique_ptr<QuicDispatcher> dispatcher_;
dispatcher_->ProcessBufferedChlos(kNumSessionsToCreatePerSocketEvent);

void QuicDispatcher::ProcessBufferedChlos(size_t max_connections_to_create) {
  // Reset the counter before starting creating connections.
  new_sessions_allowed_per_event_loop_ = max_connections_to_create;
  for (; new_sessions_allowed_per_event_loop_ > 0;
       --new_sessions_allowed_per_event_loop_) {
    QuicConnectionId server_connection_id;
    BufferedPacketList packet_list =
        buffered_packets_.DeliverPacketsForNextConnection(
            &server_connection_id);
    const std::list<BufferedPacket>& packets = packet_list.buffered_packets;
    if (packets.empty()) {
      return;
    }
    QuicConnectionId original_connection_id = server_connection_id;
    server_connection_id = MaybeReplaceServerConnectionId(server_connection_id,
                                                          packet_list.version);
    std::string alpn = SelectAlpn(packet_list.alpns);
    std::unique_ptr<QuicSession> session = CreateQuicSession(
        server_connection_id, packets.front().self_address,
        packets.front().peer_address, alpn, packet_list.version);
    if (original_connection_id != server_connection_id) {
      session->connection()->SetOriginalDestinationConnectionId(
          original_connection_id);
    }
    //创建一个新的session
    QUIC_DLOG(INFO) << "Created new session for " << server_connection_id;

    auto insertion_result = session_map_.insert(
        std::make_pair(server_connection_id, std::move(session)));
    QUIC_BUG_IF(!insertion_result.second)
        << "Tried to add a session to session_map with existing connection id: "
        << server_connection_id;
        //将数据传递给session,session之后怎么处理以后分析
    DeliverPacketsToSession(packets, insertion_result.first->second.get());
  }
}

void QuicDispatcher::DeliverPacketsToSession(
    const std::list<BufferedPacket>& packets,
    QuicSession* session) {
  for (const BufferedPacket& packet : packets) {
  //处理UDP数据包　也是通过session将数据发送出去
    session->ProcessUdpPacket(packet.self_address, packet.peer_address,
                              *(packet.packet));
  }
}
//是有session中的connection_处理的数据，
void QuicSession::ProcessUdpPacket(const QuicSocketAddress& self_address,
                                   const QuicSocketAddress& peer_address,
                                   const QuicReceivedPacket& packet) {
  connection_->ProcessUdpPacket(self_address, peer_address, packet);
}

看看connect是怎样处理数据数据包的

void QuicConnection::ProcessUdpPacket(const QuicSocketAddress& self_address,
                                      const QuicSocketAddress& peer_address,
                                      const QuicReceivedPacket& packet) {
  if (!connected_) {
    return;
  }
  QUIC_DVLOG(2) << ENDPOINT << "Received encrypted " << packet.length()
                << " bytes:" << std::endl
                << quiche::QuicheTextUtils::HexDump(quiche::QuicheStringPiece(
                       packet.data(), packet.length()));
  QUIC_BUG_IF(current_packet_data_ != nullptr)
      << "ProcessUdpPacket must not be called while processing a packet.";
  if (debug_visitor_ != nullptr) {
    debug_visitor_->OnPacketReceived(self_address, peer_address, packet);
  }
  last_size_ = packet.length();
  current_packet_data_ = packet.data();

  last_packet_destination_address_ = self_address;
  last_packet_source_address_ = peer_address;
  if (!self_address_.IsInitialized()) {
    self_address_ = last_packet_destination_address_;
  }

  if (!direct_peer_address_.IsInitialized()) {
    direct_peer_address_ = last_packet_source_address_;
  }

  if (!effective_peer_address_.IsInitialized()) {
    const QuicSocketAddress effective_peer_addr =
        GetEffectivePeerAddressFromCurrentPacket();

    // effective_peer_address_ must be initialized at the beginning of the
    // first packet processed(here). If effective_peer_addr is uninitialized,
    // just set effective_peer_address_ to the direct peer address.
    effective_peer_address_ = effective_peer_addr.IsInitialized()
                                  ? effective_peer_addr
                                  : direct_peer_address_;
  }
//更新收到的数据包和总共收到的数据大小　
  stats_.bytes_received += packet.length();
  ++stats_.packets_received;
  if (EnforceAntiAmplificationLimit()) {
    bytes_received_before_address_validation_ += last_size_;
  }

  // Ensure the time coming from the packet reader is within 2 minutes of now.
  if (std::abs((packet.receipt_time() - clock_->ApproximateNow()).ToSeconds()) >
      2 * 60) {
    QUIC_BUG << "Packet receipt time:"
             << packet.receipt_time().ToDebuggingValue()
             << " too far from current time:"
             << clock_->ApproximateNow().ToDebuggingValue();
  }
  time_of_last_received_packet_ = packet.receipt_time();
  QUIC_DVLOG(1) << ENDPOINT << "time of last received packet: "
                << packet.receipt_time().ToDebuggingValue();

  ScopedPacketFlusher flusher(this);
  if (!framer_.ProcessPacket(packet)) {
    // If we are unable to decrypt this packet, it might be
    // because the CHLO or SHLO packet was lost.
    QUIC_DVLOG(1) << ENDPOINT
                  << "Unable to process packet.  Last packet processed: "
                  << last_header_.packet_number;
    current_packet_data_ = nullptr;
    is_current_packet_connectivity_probing_ = false;

    MaybeProcessCoalescedPackets();
    return;
  }
//增加处理的数据包
  ++stats_.packets_processed;

  QUIC_DLOG_IF(INFO, active_effective_peer_migration_type_ != NO_CHANGE)
      << "sent_packet_manager_.GetLargestObserved() = "
      << sent_packet_manager_.GetLargestObserved()
      << ", highest_packet_sent_before_effective_peer_migration_ = "
      << highest_packet_sent_before_effective_peer_migration_;
  if (active_effective_peer_migration_type_ != NO_CHANGE &&
      sent_packet_manager_.GetLargestObserved().IsInitialized() &&
      (!highest_packet_sent_before_effective_peer_migration_.IsInitialized() ||
       sent_packet_manager_.GetLargestObserved() >
           highest_packet_sent_before_effective_peer_migration_)) {
    if (perspective_ == Perspective::IS_SERVER) {
      OnEffectivePeerMigrationValidated();
    }
  }

  MaybeProcessCoalescedPackets();
  MaybeProcessUndecryptablePackets();
  MaybeSendInResponseToPacket();
  SetPingAlarm();
  current_packet_data_ = nullptr;
  is_current_packet_connectivity_probing_ = false;
}

查看这个函数more_to_read = packet_reader_->ReadAndDispatchPackets

bool QuicPacketReader::ReadAndDispatchPackets(
    int fd,
    int port,
    const QuicClock& clock,
    ProcessPacketInterface* processor,
    QuicPacketCount* /*packets_dropped*/) {
  // Reset all read_results for reuse.
  for (size_t i = 0; i < read_results_.size(); ++i) {
    read_results_[i].Reset(
        /*packet_buffer_length=*/sizeof(read_buffers_[i].packet_buffer));
  }

  // Use clock.Now() as the packet receipt time, the time between packet
  // arriving at the host and now is considered part of the network delay.
  QuicTime now = clock.Now();

  size_t packets_read = socket_api_.ReadMultiplePackets(
      fd,
      BitMask64(QuicUdpPacketInfoBit::DROPPED_PACKETS,
                QuicUdpPacketInfoBit::PEER_ADDRESS,
                QuicUdpPacketInfoBit::V4_SELF_IP,
                QuicUdpPacketInfoBit::V6_SELF_IP,
                QuicUdpPacketInfoBit::RECV_TIMESTAMP, QuicUdpPacketInfoBit::TTL,
                QuicUdpPacketInfoBit::GOOGLE_PACKET_HEADER),
      &read_results_);
  for (size_t i = 0; i < packets_read; ++i) {
    auto& result = read_results_[i];
    if (!result.ok) {
      QUIC_CODE_COUNT(quic_packet_reader_read_failure);
      continue;
    }
    //获取到对端的ip地址
    if (!result.packet_info.HasValue(QuicUdpPacketInfoBit::PEER_ADDRESS)) {
      QUIC_BUG << "Unable to get peer socket address.";
      continue;
    }

    QuicSocketAddress peer_address =
        result.packet_info.peer_address().Normalized();
　//获取本机地址
    QuicIpAddress self_ip = GetSelfIpFromPacketInfo(
        result.packet_info, peer_address.host().IsIPv6());
    if (!self_ip.IsInitialized()) {
      QUIC_BUG << "Unable to get self IP address.";
      continue;
    }
//查看包是否有生存时间
    bool has_ttl = result.packet_info.HasValue(QuicUdpPacketInfoBit::TTL);
    int ttl = has_ttl ? result.packet_info.ttl() : 0;
    if (!has_ttl) {
      QUIC_CODE_COUNT(quic_packet_reader_no_ttl);
    }
//判断是否有包的头部　如果有的话就保留下来
    char* headers = nullptr;
    size_t headers_length = 0;
    if (result.packet_info.HasValue(
            QuicUdpPacketInfoBit::GOOGLE_PACKET_HEADER)) {
      headers = result.packet_info.google_packet_headers().buffer;
      headers_length = result.packet_info.google_packet_headers().buffer_len;
    } else {
      QUIC_CODE_COUNT(quic_packet_reader_no_google_packet_header);
    }
//设置一个receive   
    QuicReceivedPacket packet(
        result.packet_buffer.buffer, result.packet_buffer.buffer_len, now,
        /*owns_buffer=*/false, ttl, has_ttl, headers, headers_length,
        /*owns_header_buffer=*/false);

    QuicSocketAddress self_address(self_ip, port);
    //数据包最终还是交给dispatcher来处理
    processor->ProcessPacket(self_address, peer_address, packet);
  }

  // We may not have read all of the packets available on the socket.
  return packets_read == kNumPacketsPerReadMmsgCall;
}

看看dispatcher的processPacket函数、处理数据包，怎么处理的，不详

void QuicDispatcher::ProcessPacket(const QuicSocketAddress& self_address,
                                   const QuicSocketAddress& peer_address,
                                   const QuicReceivedPacket& packet) {
  //收到一个加密数据包
  QUIC_DVLOG(2) << "Dispatcher received encrypted " << packet.length()
                << " bytes:" << std::endl
                << quiche::QuicheTextUtils::HexDump(quiche::QuicheStringPiece(
                       packet.data(), packet.length()));
  ReceivedPacketInfo packet_info(self_address, peer_address, packet);
  std::string detailed_error;
  bool retry_token_present;
  quiche::QuicheStringPiece retry_token;
  const QuicErrorCode error = QuicFramer::ParsePublicHeaderDispatcher(
      packet, expected_server_connection_id_length_, &packet_info.form,
      &packet_info.long_packet_type, &packet_info.version_flag,
      &packet_info.use_length_prefix, &packet_info.version_label,
      &packet_info.version, &packet_info.destination_connection_id,
      &packet_info.source_connection_id, &retry_token_present, &retry_token,
      &detailed_error);
  if (error != QUIC_NO_ERROR) {
    // Packet has framing error.
    SetLastError(error);
    QUIC_DLOG(ERROR) << detailed_error;
    return;
  }
  if (packet_info.destination_connection_id.length() !=
          expected_server_connection_id_length_ &&
      !should_update_expected_server_connection_id_length_ &&
      packet_info.version.IsKnown() &&
      !packet_info.version.AllowsVariableLengthConnectionIds()) {
    SetLastError(QUIC_INVALID_PACKET_HEADER);
    QUIC_DLOG(ERROR) << "Invalid Connection Id Length";
    return;
  }

  if (packet_info.version_flag && IsSupportedVersion(packet_info.version)) {
    if (!QuicUtils::IsConnectionIdValidForVersion(
            packet_info.destination_connection_id,
            packet_info.version.transport_version)) {
      SetLastError(QUIC_INVALID_PACKET_HEADER);
      QUIC_DLOG(ERROR)
          << "Invalid destination connection ID length for version";
      return;
    }
    if (packet_info.version.SupportsClientConnectionIds() &&
        !QuicUtils::IsConnectionIdValidForVersion(
            packet_info.source_connection_id,
            packet_info.version.transport_version)) {
      SetLastError(QUIC_INVALID_PACKET_HEADER);
      QUIC_DLOG(ERROR) << "Invalid source connection ID length for version";
      return;
    }
  }

  if (should_update_expected_server_connection_id_length_) {
    expected_server_connection_id_length_ =
        packet_info.destination_connection_id.length();
  }

  if (MaybeDispatchPacket(packet_info)) {
    // Packet has been dropped or successfully dispatched, stop processing.
    return;
  }
  ProcessHeader(&packet_info);
}

下面开始分析MyQuicToyServer的HandleEvent

server.HandleEvent();
//进入这个类查看函数
void  MyQuicToyServer::HandleEvent(){
    if(server_){
        server_->WaitForEvents();  
    }
}
void MyQuicServer::WaitForEvents() {
//获取当前线程ID
  context_id_=base::PlatformThread::CurrentId();  
  //等待当前事件的发生
  epoll_server_.WaitForEventsAndExecuteCallbacks();
    std::deque<std::unique_ptr<QueuedTask>> tasks;
    {
      //交换
        QuicWriterMutexLock lock(&task_mutex_);
        tasks.swap(queued_tasks_);//交换queue队列的任务
    }
    while(!tasks.empty()){
        tasks.front()->Run();
        tasks.pop_front();
    }   
}

寻找一下这个queued_tasks_

//这一块没太看明白　还需等待
void ClientThread::PostInnerTask(std::unique_ptr<QueuedTask> task){
    QuicWriterMutexLock lock(&task_mutex_);
    queued_tasks_.push_back(std::move(task));
}