(01)Webrtc::Fec与Nack的二三事

  写在前面：要理解Fec与Nack逻辑，我喜欢先从接受端看， 理解了Fec与Nack是如何被使用的，才能更好的明白不同的机制应该怎么用，在什么场合用。

 

  更新丢包逻辑

void PacketBuffer::UpdateMissingPackets(uint16_t seq_num) {
  if (!newest_inserted_seq_num_)
    newest_inserted_seq_num_ = seq_num;

  const int kMaxPaddingAge = 1000;
  if (AheadOf(seq_num, *newest_inserted_seq_num_)) {
    uint16_t old_seq_num = seq_num - kMaxPaddingAge;
    auto erase_to = missing_packets_.lower_bound(old_seq_num);
    missing_packets_.erase(missing_packets_.begin(), erase_to);

    // Guard against inserting a large amount of missing packets if there is a
    // jump in the sequence number.
    if (AheadOf(old_seq_num, *newest_inserted_seq_num_)) {
      *newest_inserted_seq_num_ = old_seq_num;
    }

    ++*newest_inserted_seq_num_;
    while (AheadOf(seq_num, *newest_inserted_seq_num_)) {
      missing_packets_.insert(*newest_inserted_seq_num_);
      ++*newest_inserted_seq_num_;
    }
  } else {
    missing_packets_.erase(seq_num);
  }
}

  1.AheadOf(A, B)可以简单理解为A>B。实际环形数据的比较与环形存储空间size有关，相当与（A > B) xor (abs(A - B) > size / 2)。

  2.首先判断当前包序号seq_num是否在之前插入的包序号newest_inserted_seq_num之后，如果不是，证明是之前的丢包，走else逻辑。如果是，则开始判断是否有丢包。

  3.删除与当前包序号相差超过1000的丢包序号，因为太老了。如果seq_num - 1000 > newest_inserted_seq_num，证明中间丢包太多，只从seq_num - 1000开始插入丢包序号。

  4.newest_inserted_seq_num到seq_num之间的全部包被认定为丢包。

举例：收包1000， 1001， 1002， 1005，1003……收到1005时认定1003/1004丢包，收到1003时删除丢包列表中的1003，只剩1004。

注意：packet_buffer的missing_packets_在webrtc中虽然做了更新，但除了H264判断IPPP依赖是否有丢依赖帧之外，并没有被使用，所以这里其实并不影响实际的网络体验。

 

  判断收到数据包时是否可能组成一帧

bool PacketBuffer::PotentialNewFrame(uint16_t seq_num) const {
  size_t index = seq_num % buffer_.size();
  int prev_index = index > 0 ? index - 1 : buffer_.size() - 1;
  const auto& entry = buffer_[index];
  const auto& prev_entry = buffer_[prev_index];

  if (entry == nullptr)
    return false;
  if (entry->seq_num != seq_num)
    return false;
  if (entry->is_first_packet_in_frame())
    return true;
  if (prev_entry == nullptr)
    return false;
  if (prev_entry->seq_num != static_cast<uint16_t>(entry->seq_num - 1))
    return false;
  if (prev_entry->timestamp != entry->timestamp)
    return false;
  if (prev_entry->continuous)
    return true;

  return false;
}

  判断是否可能组成一个完整帧的条件有两个：1.该包为帧的地一个包。2.包连续，前一包存在且时间戳相同。可以看到，这里用于判断的是buffer_的实际数据，而非missing_packets_。buffer_在执行PacketBuffer::InsertPacket时被更新。

  注意这里的continuous，有两个位置会设置这个值：InsertPacket的时候被置为false，或是通过PotentialNewFrame检测后在FindFrames中被置为true。也就是说，一个包的continuous置为true，唯一的途径为通过PotentialNewFrame检测。于是一帧里的第二个包被置为true，依赖于收到第一个包，走is_first_packet_in_frame() == true分支，而第N个包被置为true依赖于第N-1个包被置为true。所以一个包的上一个包的continuous标记为true，就表示这个包所属帧，从第一个包到当前包都是连续的。

 

  寻找完整帧

std::vector<std::unique_ptr<PacketBuffer::Packet>> PacketBuffer::FindFrames(
    uint16_t seq_num) {
  std::vector<std::unique_ptr<PacketBuffer::Packet>> found_frames;

  for (size_t i = 0; i < buffer_.size() && PotentialNewFrame(seq_num); ++i) {
    size_t index = seq_num % buffer_.size();
    buffer_[index]->continuous = true;

    // If all packets of the frame is continuous, find the first packet of the
    // frame and add all packets of the frame to the returned packets.
    if (buffer_[index]->is_last_packet_in_frame()) {
      uint16_t start_seq_num = seq_num;

      // Find the start index by searching backward until the packet with
      // the |frame_begin| flag is set.
      int start_index = index;
      size_t tested_packets = 0;
      int64_t frame_timestamp = buffer_[start_index]->timestamp;

      // Identify H.264 keyframes by means of SPS, PPS, and IDR.
      bool is_h264 = buffer_[start_index]->codec() == kVideoCodecH264;
      bool has_h264_sps = false;
      bool has_h264_pps = false;
      bool has_h264_idr = false;
      bool is_h264_keyframe = false;
      int idr_width = -1;
      int idr_height = -1;
      while (true) {
        ++tested_packets;

        if (!is_h264 && buffer_[start_index]->is_first_packet_in_frame())
          break;

        if (is_h264) {
          // ture: RTC_LOG(LS_ERROR) << "kiki-h264";
          const auto* h264_header = absl::get_if<RTPVideoHeaderH264>(
              &buffer_[start_index]->video_header.video_type_header);
          if (!h264_header || h264_header->nalus_length >= kMaxNalusPerPacket)
            return found_frames;

          for (size_t j = 0; j < h264_header->nalus_length; ++j) {
            if (h264_header->nalus[j].type == H264::NaluType::kSps) {
              has_h264_sps = true;
            } else if (h264_header->nalus[j].type == H264::NaluType::kPps) {
              has_h264_pps = true;
            } else if (h264_header->nalus[j].type == H264::NaluType::kIdr) {
              has_h264_idr = true;
            }
          }
          if ((sps_pps_idr_is_h264_keyframe_ && has_h264_idr && has_h264_sps &&
               has_h264_pps) ||
              (!sps_pps_idr_is_h264_keyframe_ && has_h264_idr)) {
            is_h264_keyframe = true;
            // Store the resolution of key frame which is the packet with
            // smallest index and valid resolution; typically its IDR or SPS
            // packet; there may be packet preceeding this packet, IDR's
            // resolution will be applied to them.
            if (buffer_[start_index]->width() > 0 &&
                buffer_[start_index]->height() > 0) {
              idr_width = buffer_[start_index]->width();
              idr_height = buffer_[start_index]->height();
            }
          }
        }

        if (tested_packets == buffer_.size())
          break;

        start_index = start_index > 0 ? start_index - 1 : buffer_.size() - 1;

        // In the case of H264 we don't have a frame_begin bit (yes,
        // |frame_begin| might be set to true but that is a lie). So instead
        // we traverese backwards as long as we have a previous packet and
        // the timestamp of that packet is the same as this one. This may cause
        // the PacketBuffer to hand out incomplete frames.
        // See: https://bugs.chromium.org/p/webrtc/issues/detail?id=7106
        if (is_h264 && (buffer_[start_index] == nullptr ||
                        buffer_[start_index]->timestamp != frame_timestamp)) {
          break;
        }

        --start_seq_num;
      }

      if (is_h264) {
        // Warn if this is an unsafe frame.
        if (has_h264_idr && (!has_h264_sps || !has_h264_pps)) {
          RTC_LOG(LS_WARNING)
              << "Received H.264-IDR frame "
                 "(SPS: "
              << has_h264_sps << ", PPS: " << has_h264_pps << "). Treating as "
              << (sps_pps_idr_is_h264_keyframe_ ? "delta" : "key")
              << " frame since WebRTC-SpsPpsIdrIsH264Keyframe is "
              << (sps_pps_idr_is_h264_keyframe_ ? "enabled." : "disabled");
        }

        // Now that we have decided whether to treat this frame as a key frame
        // or delta frame in the frame buffer, we update the field that
        // determines if the RtpFrameObject is a key frame or delta frame.
        const size_t first_packet_index = start_seq_num % buffer_.size();
        if (is_h264_keyframe) {
          buffer_[first_packet_index]->video_header.frame_type =
              VideoFrameType::kVideoFrameKey;
          if (idr_width > 0 && idr_height > 0) {
            // IDR frame was finalized and we have the correct resolution for
            // IDR; update first packet to have same resolution as IDR.
            buffer_[first_packet_index]->video_header.width = idr_width;
            buffer_[first_packet_index]->video_header.height = idr_height;
          }
        } else {
          buffer_[first_packet_index]->video_header.frame_type =
              VideoFrameType::kVideoFrameDelta;
        }

        // With IPPP, if this is not a keyframe, make sure there are no gaps
        // in the packet sequence numbers up until this point.
        const uint8_t h264tid =
            buffer_[start_index] != nullptr
                ? buffer_[start_index]->video_header.frame_marking.temporal_id
                : kNoTemporalIdx;
        if (h264tid == kNoTemporalIdx && !is_h264_keyframe &&
            missing_packets_.upper_bound(start_seq_num) !=
                missing_packets_.begin()) {
          return found_frames;
        }
      }

      const uint16_t end_seq_num = seq_num + 1;
      // Use uint16_t type to handle sequence number wrap around case.
      uint16_t num_packets = end_seq_num - start_seq_num;
      found_frames.reserve(found_frames.size() + num_packets);
      for (uint16_t i = start_seq_num; i != end_seq_num; ++i) {
        std::unique_ptr<Packet>& packet = buffer_[i % buffer_.size()];
        RTC_DCHECK(packet);
        RTC_DCHECK_EQ(i, packet->seq_num);
        // Ensure frame boundary flags are properly set.
        packet->video_header.is_first_packet_in_frame = (i == start_seq_num);
        packet->video_header.is_last_packet_in_frame = (i == seq_num);
        found_frames.push_back(std::move(packet));
      }

      missing_packets_.erase(missing_packets_.begin(),
                             missing_packets_.upper_bound(seq_num));
    }
    ++seq_num;
  }
  return found_frames;
}

  1.正常网络条件下的视频包，都会通过PotentialNewFrame检测，然后因为不是帧的最后一包，直接seq_num++，下一次循环entry == nullptr而退出循环。seq_num++的原因是，新插入的包可能是帧间丢失的最后一包，例如一帧由1000-1006包组成，只有1003包丢失，则收到1003包时会首先通过PotentialNewFrame检测，然后因为不是帧的最后一包，不断seq_num++，将1004/1005/1006包的continuous标志置为true，并从1006包开始组帧搜索。

  2.如果是帧的最后一包，开始反向寻找帧的第一包。对于h264，判断方式为a.前一包为空，b.前一包的时间戳不同。注释说h264的frame begin位是假的，所以不用改标志作为判断。不是很理解，实测标志位是有效且准确的。标志位通过video_rtp_depacketizer_h264的ParseFuaNalu函数，first_fragment = (rtp_payload.cdata()[1] & kSBit) > 0判断。

// In the case of H264 we don't have a frame_begin bit (yes,
// |frame_begin| might be set to true but that is a lie). So instead
// we traverese backwards as long as we have a previous packet and
// the timestamp of that packet is the same as this one. This may cause
// the PacketBuffer to hand out incomplete frames.
// See: https://bugs.chromium.org/p/webrtc/issues/detail?id=7106

  3.对于h264，如果missing_packets_.upper_bound(start_seq_num) != missing_packets_.begin()并且当前不是I帧，说明当前虽然能组成P帧，但IPPP依赖缺失，直接返回。一个问题是导致卡顿，因为当前画面本来是可以播放的，只是有可能花屏。另一个问题是导致丢帧，因为当前帧所有包已经收齐，接下来再收的包都不会触发这一帧的组帧搜索，而这一帧本身并没有被返回。（前面说的不对，FindFrames返回不止一帧，当前面的依赖帧收全后，会触发这一帧的返回）这里可以优化，即使前面P帧缺失也返回当前帧，但可能会导致部分花屏，特别对于GOP20秒且网络不好的情况，可能会一直在花屏状态。

  4.从日志来看，h264收到的每个I帧都是IDR，包含SPS/PPS。

  5.nalus_length代表包里包含几帧，对于视频来说，h264为fu-a分包，即多个包组合一个帧。直到一帧的起始包，才会看到nalus_length为1，其它为0。

  6.h264tid一直是kNoTemporalIdx，不太理解这是什么标志。

  7.将可以组帧的所有包返回，并更新missing_packets_，既然已经组帧，那么seq_num之前的丢包全部不要了。这里很重要，如果没有这个，做了第3条红字部分优化，可能会导致退帧现象。std::move会清空buffer_对应位置数据。

 

  NACK

bool NackModule::RemovePacketsUntilKeyFrame() {
  while (!keyframe_list_.empty()) {
    auto it = nack_list_.lower_bound(*keyframe_list_.begin());

    if (it != nack_list_.begin()) {
      // We have found a keyframe that actually is newer than at least one
      // packet in the nack list.
      nack_list_.erase(nack_list_.begin(), it);
      return true;
    }

    // If this keyframe is so old it does not remove any packets from the list,
    // remove it from the list of keyframes and try the next keyframe.
    keyframe_list_.erase(keyframe_list_.begin());
  }
  return false;
}

void NackModule::AddPacketsToNack(uint16_t seq_num_start,
                                  uint16_t seq_num_end) {
  // Remove old packets.
  auto it = nack_list_.lower_bound(seq_num_end - kMaxPacketAge);
  nack_list_.erase(nack_list_.begin(), it);

  // If the nack list is too large, remove packets from the nack list until
  // the latest first packet of a keyframe. If the list is still too large,
  // clear it and request a keyframe.
  uint16_t num_new_nacks = ForwardDiff(seq_num_start, seq_num_end);
  if (nack_list_.size() + num_new_nacks > kMaxNackPackets) {
    while (RemovePacketsUntilKeyFrame() &&
           nack_list_.size() + num_new_nacks > kMaxNackPackets) {
    }

    if (nack_list_.size() + num_new_nacks > kMaxNackPackets) {
      nack_list_.clear();
      keyframe_request_sender_->RequestKeyFrame();
      return;
    }
  }

  for (uint16_t seq_num = seq_num_start; seq_num != seq_num_end; ++seq_num) {
    // Do not send nack for packets that are already recovered by FEC or RTX
    if (recovered_list_.find(seq_num) != recovered_list_.end())
      continue;
    NackInfo nack_info(seq_num, seq_num + WaitNumberOfPackets(0.5),
                       clock_->TimeInMilliseconds());
    RTC_DCHECK(nack_list_.find(seq_num) == nack_list_.end());
    nack_list_[seq_num] = nack_info;
  }
}

  1.nack_list_维护丢包seq_num，keyframe_list_维护已收到的I帧seq_num。

  2.当nacks数目超过kMaxNackPackets，首先不断调用RemovePacketUntilKeyFrame，指导返回结果为false或者数目不再超过kMaxNackPackets。

  3.RemovePacketUntilKeyFrame逻辑：

      a.如果nack_list_中有早于keyframe_list_最早关键帧的seq_num，删除这些包序号直到最早关键帧seq_num，返回true。

      b.如不符合条件a，删除最早关键帧seq_num，并再次进行判断。

      c.当删光keyframe_list_后返回false。

  4.如果删光keyframe_list_后，nacks数目依然超标，清空nack_list_，并请求重发关键帧。

  5.循环每一个丢包的seq_num，由seq_num生成NackInfo信息，插入nack_list_中。注意因为在NACK的OnReceivedPacket函数中，写死了”bool is_retransmitted = true;“，不会调用UpdateReorderingStatistics，所以WaitNumberOfPackets一直返回0。NackInfo的seq_num与send_at_seq_num均为丢包的seq_num。

  6.可以看到，在逻辑中，如果包已经被fec恢复，就不会加入nack_list_列表中。但因为ulpfec传输视频包与fec包的顺序，大部分情况下丢包检测先于fec包到达，这里实际没什么作用。

 

  收包的NACK回调

int NackModule::OnReceivedPacket(uint16_t seq_num,
                                 bool is_keyframe,
                                 bool is_recovered) {
  rtc::CritScope lock(&crit_);
  // TODO(philipel): When the packet includes information whether it is
  //                 retransmitted or not, use that value instead. For
  //                 now set it to true, which will cause the reordering
  //                 statistics to never be updated.
  bool is_retransmitted = true;

  if (!initialized_) {
    newest_seq_num_ = seq_num;
    if (is_keyframe)
      keyframe_list_.insert(seq_num);
    initialized_ = true;
    return 0;
  }

  // Since the |newest_seq_num_| is a packet we have actually received we know
  // that packet has never been Nacked.
  if (seq_num == newest_seq_num_)
    return 0;

  if (AheadOf(newest_seq_num_, seq_num)) {
    // An out of order packet has been received.
    auto nack_list_it = nack_list_.find(seq_num);
    int nacks_sent_for_packet = 0;
    if (nack_list_it != nack_list_.end()) {
      nacks_sent_for_packet = nack_list_it->second.retries;
      nack_list_.erase(nack_list_it);
    }
    if (!is_retransmitted)
      UpdateReorderingStatistics(seq_num);
    return nacks_sent_for_packet;
  }

  // Keep track of new keyframes.
  if (is_keyframe)
    keyframe_list_.insert(seq_num);

  // And remove old ones so we don't accumulate keyframes.
  auto it = keyframe_list_.lower_bound(seq_num - kMaxPacketAge);
  if (it != keyframe_list_.begin())
    keyframe_list_.erase(keyframe_list_.begin(), it);

  if (is_recovered) {
    recovered_list_.insert(seq_num);

    // Remove old ones so we don't accumulate recovered packets.
    auto it = recovered_list_.lower_bound(seq_num - kMaxPacketAge);
    if (it != recovered_list_.begin())
      recovered_list_.erase(recovered_list_.begin(), it);

    // Do not send nack for packets recovered by FEC or RTX.
    return 0;
  }

  AddPacketsToNack(newest_seq_num_ + 1, seq_num);
  newest_seq_num_ = seq_num;

  // Are there any nacks that are waiting for this seq_num.
  std::vector<uint16_t> nack_batch = GetNackBatch(kSeqNumOnly);
  if (!nack_batch.empty()) {
    // This batch of NACKs is triggered externally; the initiator can
    // batch them with other feedback messages.
    nack_sender_->SendNack(nack_batch, /*buffering_allowed=*/true);
  }

  return 0;
}

  1.当新收到的seq_num存在于nack_list_中，表明这个包已经收到，不需要重传，从nack_list_中移除。返回这个包被重传次数。

  2.is_keyframe表示收到的包是否输入关键帧，is_recovered表示收到的包是否是从fec恢复回来的，分别处理keyframe_list_及recovered_list_。如果是恢复包，不触发丢包更新直接返回。

  3.AddPacketsToNack更新丢包列表。

std::vector<uint16_t> NackModule::GetNackBatch(NackFilterOptions options) {
  bool consider_seq_num = options != kTimeOnly;
  bool consider_timestamp = options != kSeqNumOnly;
  Timestamp now = clock_->CurrentTime();
  std::vector<uint16_t> nack_batch;
  auto it = nack_list_.begin();
  while (it != nack_list_.end()) {
    TimeDelta resend_delay = TimeDelta::Millis(rtt_ms_);
    if (backoff_settings_) {
      resend_delay =
          std::max(resend_delay, backoff_settings_->min_retry_interval);
      if (it->second.retries > 1) {
        TimeDelta exponential_backoff =
            std::min(TimeDelta::Millis(rtt_ms_), backoff_settings_->max_rtt) *
            std::pow(backoff_settings_->base, it->second.retries - 1);
        resend_delay = std::max(resend_delay, exponential_backoff);
      }
    }

    bool delay_timed_out =
        now.ms() - it->second.created_at_time >= send_nack_delay_ms_;
    bool nack_on_rtt_passed =
        now.ms() - it->second.sent_at_time >= resend_delay.ms();
    bool nack_on_seq_num_passed =
        it->second.sent_at_time == -1 &&
        AheadOrAt(newest_seq_num_, it->second.send_at_seq_num);
    if (delay_timed_out && ((consider_seq_num && nack_on_seq_num_passed) ||
                            (consider_timestamp && nack_on_rtt_passed))) {
      nack_batch.emplace_back(it->second.seq_num);
      ++it->second.retries;
      it->second.sent_at_time = now.ms();
      if (it->second.retries >= kMaxNackRetries) {
        RTC_LOG(LS_WARNING) << "Sequence number " << it->second.seq_num
                            << " removed from NACK list due to max retries.";
        it = nack_list_.erase(it);
      } else {
        ++it;
      }
      continue;
    }
    ++it;
  }
  return nack_batch;
}

  4.GetNackBatch函数对于nack_list_中满足一定条件的包，将其加入nack_batch一并返回，如果重传次数过多，从nack_list_中移除。这个函数与SendNack有两种被触发的机制，一是每次nack module收到包的回调（NackModule::OnReceivedPacket），模式为kSeqNumOnly，二是nack module轮询，每隔20ms一次（NackModule::Process），模式为kTimeOnly。值得注意的是，NackInfo的sent_at_time并不是-1，所以轮询方式只会根据delay_timed_out成立执行逻辑。限制条件的目的大概是为了防止前一个nack重传发送端还没处理完就又发了新的，同一个seq_num明明发一次就可以解决但被发了多次。

  5.SendNack调用：RtpVideoStreamReceiver::RtcpFeedbackBuffer::SendNack。这个函数首先将batch中的seq_num加入nack_sequence_numbers_中，然后根据buffering_allowed决定是否发送nack包。所以在回调中，只会做插入操作而不会发送nack包，只有在轮询中才会实际发送nack的rtcp请求。这里也可以优化，将OnReceivedPacket中的buffering_allowed改为true，以保证重传/组帧更快的发生，降低延迟。SendBufferedRtcpFeedback -> VideoReceiveStream::SendNack -> RtpVideoStreamReceiver::RequestPacketRetransmit -> ModuleRtpRtcpImpl::SendNack发送对应的rtcp包。

  6.补充：rtp_video_stream_receiver.cc初始化rtcp_feedback_buffer_的KeyFrameRequestSender及LossNotificationSender为自己，而NackSender为video_receive_stream.cc的VideoReceiveStream类。绕的要命，没理解这个设计模式。

 

  FEC

  1.fec维护两个重要队列，recovered_packets及received_fec_packets_。recovered_packets里保存收到的视频包，以及被fec恢复的包。received_fec_packets_里保存收到的fec包，每个fec包还包含指向所保护数据包的指针。

void ForwardErrorCorrection::DecodeFec(const ReceivedPacket& received_packet,
                                       RecoveredPacketList* recovered_packets) {
  RTC_DCHECK(recovered_packets);

  const size_t max_media_packets = fec_header_reader_->MaxMediaPackets();
  if (recovered_packets->size() == max_media_packets) {
    const RecoveredPacket* back_recovered_packet =
        recovered_packets->back().get();

    if (received_packet.ssrc == back_recovered_packet->ssrc) {
      const unsigned int seq_num_diff =
          MinDiff(received_packet.seq_num, back_recovered_packet->seq_num);
      if (seq_num_diff > max_media_packets) {
        // A big gap in sequence numbers. The old recovered packets
        // are now useless, so it's safe to do a reset.
        RTC_LOG(LS_INFO) << "Big gap in media/ULPFEC sequence numbers. No need "
                            "to keep the old packets in the FEC buffers, thus "
                            "resetting them.";
        ResetState(recovered_packets);
      }
    }
  }

  InsertPacket(received_packet, recovered_packets);
  AttemptRecovery(recovered_packets);
}

  2.首先做一些过期检测，如果当前的包的seq_num与最后一个收到或恢复的视频包的seq_num差距超过了fec所保护的数据包数量上限，说明之前的包无论如何不可能被恢复了，清空recovered_packets及received_fec_packets_。

void ForwardErrorCorrection::InsertPacket(
    const ReceivedPacket& received_packet,
    RecoveredPacketList* recovered_packets) {

  if (!received_fec_packets_.empty() &&
      received_packet.ssrc == received_fec_packets_.front()->ssrc) {

    auto it = received_fec_packets_.begin();
    while (it != received_fec_packets_.end()) {
      uint16_t seq_num_diff = MinDiff(received_packet.seq_num, (*it)->seq_num);
      if (seq_num_diff > 0x3fff) {
        it = received_fec_packets_.erase(it);
      } else {
        // No need to keep iterating, since |received_fec_packets_| is sorted.
        break;
      }
    }
  }

  if (received_packet.is_fec) {
    InsertFecPacket(*recovered_packets, received_packet);
  } else {
    InsertMediaPacket(recovered_packets, received_packet);
  }

  DiscardOldRecoveredPackets(recovered_packets);
}

  3.对于fec包，a.通过mask解析保护的所有视频包序列号。b.如果序列号存在于recovered_packets中，将对应的指针指向数据包。c.将fec包放入received_fec_packets_。

  4.对于视频包，a.首先放入recovered_packet。b.对于received_fec_packets_中每一个fec包，如果保护的数据包包含该视频包，更新指针。

  5.丢弃用不到的数据包：例如fec最多只保护48个包（看协议，应该最多只能保护24个包，不知道这里为什么设置为48，long mask是1:48bits mask），而recovered_packets中包数超过这个阈值，多出来的显然不可能被用到。

void ForwardErrorCorrection::AttemptRecovery(
    RecoveredPacketList* recovered_packets) {
  auto fec_packet_it = received_fec_packets_.begin();
  while (fec_packet_it != received_fec_packets_.end()) {
    int packets_missing = NumCoveredPacketsMissing(**fec_packet_it);
    if (packets_missing == 1) {
      std::unique_ptr<RecoveredPacket> recovered_packet(new RecoveredPacket());
      recovered_packet->pkt = nullptr;
      if (!RecoverPacket(**fec_packet_it, recovered_packet.get())) {
        fec_packet_it = received_fec_packets_.erase(fec_packet_it);
        continue;
      }
      auto* recovered_packet_ptr = recovered_packet.get();
      recovered_packets->push_back(std::move(recovered_packet));
      recovered_packets->sort(SortablePacket::LessThan());
      UpdateCoveringFecPackets(*recovered_packet_ptr);
      DiscardOldRecoveredPackets(recovered_packets);
      fec_packet_it = received_fec_packets_.erase(fec_packet_it);
      fec_packet_it = received_fec_packets_.begin();
    } else if (packets_missing == 0) {
      fec_packet_it = received_fec_packets_.erase(fec_packet_it);
    } else {
      fec_packet_it++;
    }
  }
}

  6.对于received_fec_packets_中的每一个fec包，首先确认其所保护的包里有多少丢包，判断方式即查看每一个保护包的指针是否为空。通过xor方式只能恢复一个丢包，例如：a xor b xor c = fec -> a = b xor c xor fec。所以如果丢包数超过1个，无法恢复丢包，如果丢包数为0，则fec所保护的所有包均已收到或恢复，fec包不再有意义，丢弃。只有丢包数为1时，通过xor恢复数据包，并将恢复的包，放入recovered_packets中。

 

  FEC解包逻辑

  1.RED包会调用两次ReceivePacket，第一次解析剥离RED头。

void RtpVideoStreamReceiver::NotifyReceiverOfEmptyPacket(uint16_t seq_num) {
  {
    rtc::CritScope lock(&reference_finder_lock_);
    reference_finder_->PaddingReceived(seq_num);
  }
  OnInsertedPacket(packet_buffer_.InsertPadding(seq_num));
  if (nack_module_) {
    nack_module_->OnReceivedPacket(seq_num, /* is_keyframe = */ false,
                                   /* is _recovered = */ false);
  }
  if (loss_notification_controller_) {
    // TODO(bugs.webrtc.org/10336): Handle empty packets.
    RTC_LOG(LS_WARNING)
        << "LossNotificationController does not expect empty packets.";
  }
}

PacketBuffer::InsertResult PacketBuffer::InsertPadding(uint16_t seq_num) {
  PacketBuffer::InsertResult result;
  rtc::CritScope lock(&crit_);
  UpdateMissingPackets(seq_num);
  result.packets = FindFrames(static_cast<uint16_t>(seq_num + 1));
  return result;
}

  2.Fec包会触发NotifyReceiverOfEmptyPacket函数。代码注释给的解释是“Notify video_receiver about received FEC packets to avoid NACKing these packets.”。

其中InsertPadding主要做几件事：更新missing_packets_，以及看有没有可能组帧。注意这里传入FindFrames里的是seq_num+1，无论后一个包是新一帧的起始包还是另一个fec包，都不可能找到完整帧（如果是新一帧并且收到全部包，则早已返回，这里PotentialNewFrame也只会entry == nullptr），不是很理解实际意义。OnInsertedPacket会调用AssembleFrame，将属于同一帧的所有包组成这里results.packets为空，当然也不会有操作。这里还有一件重要的事，调用RtpFrameReferenceFinder::PaddingReceived，这个RtpFrameReferenceFinder是用于判断帧依赖的，并且在引入fec后，对于h264的依赖判断有问题，后面仔细分析一下。另外调用nack的收包回调，更新丢包列表。从这里看，padding包/fec包与普通包在nack模块看来，都是一样的，所以fec包丢失也会重传。

  Note: 基于这个原因，Ulpfec一定程度会恶化网页版的体验，因为无法在接收端做对应优化。解决办法是使用独立的传输通路与独立的seq_num，即flexfec。

bool UlpfecReceiverImpl::AddReceivedRedPacket(
    const RtpPacketReceived& rtp_packet,
    uint8_t ulpfec_payload_type) {
  if (rtp_packet.Ssrc() != ssrc_) {
    RTC_LOG(LS_WARNING)
        << "Received RED packet with different SSRC than expected; dropping.";
    return false;
  }
  if (rtp_packet.size() > IP_PACKET_SIZE) {
    RTC_LOG(LS_WARNING) << "Received RED packet with length exceeds maximum IP "
                           "packet size; dropping.";
    return false;
  }
  rtc::CritScope cs(&crit_sect_);

  static constexpr uint8_t kRedHeaderLength = 1;

  if (rtp_packet.payload_size() == 0) {
    RTC_LOG(LS_WARNING) << "Corrupt/truncated FEC packet.";
    return false;
  }

  // Remove RED header of incoming packet and store as a virtual RTP packet.
  auto received_packet =
      std::make_unique<ForwardErrorCorrection::ReceivedPacket>();
  received_packet->pkt = new ForwardErrorCorrection::Packet();

  // Get payload type from RED header and sequence number from RTP header.
  uint8_t payload_type = rtp_packet.payload()[0] & 0x7f;
  received_packet->is_fec = payload_type == ulpfec_payload_type;
  received_packet->is_recovered = rtp_packet.recovered();
  received_packet->ssrc = rtp_packet.Ssrc();
  received_packet->seq_num = rtp_packet.SequenceNumber();

  // RTC_LOG(LS_ERROR) << "kiki-fec/packet " << received_packet->seq_num << " is fec/recovered? " << received_packet->is_fec << " " << received_packet->is_recovered;

  if (rtp_packet.payload()[0] & 0x80) {
    // f bit set in RED header, i.e. there are more than one RED header blocks.
    // WebRTC never generates multiple blocks in a RED packet for FEC.
    RTC_LOG(LS_WARNING) << "More than 1 block in RED packet is not supported.";
    return false;
  }

  ++packet_counter_.num_packets;
  packet_counter_.num_bytes += rtp_packet.size();
  if (packet_counter_.first_packet_time_ms == -1) {
    packet_counter_.first_packet_time_ms = rtc::TimeMillis();
  }

  if (received_packet->is_fec) {
    ++packet_counter_.num_fec_packets;
    // everything behind the RED header
    received_packet->pkt->data =
        rtp_packet.Buffer().Slice(rtp_packet.headers_size() + kRedHeaderLength,
                                  rtp_packet.payload_size() - kRedHeaderLength);
  } else {
    auto red_payload = rtp_packet.payload().subview(kRedHeaderLength);
    received_packet->pkt->data.EnsureCapacity(rtp_packet.headers_size() +
                                              red_payload.size());
    // Copy RTP header.
    received_packet->pkt->data.SetData(rtp_packet.data(),
                                       rtp_packet.headers_size());
    // Set payload type.
    received_packet->pkt->data[1] &= 0x80;          // Reset RED payload type.
    received_packet->pkt->data[1] += payload_type;  // Set media payload type.
    // Copy payload data.
    received_packet->pkt->data.AppendData(red_payload.data(),
                                          red_payload.size());
  }

  if (received_packet->pkt->data.size() > 0) {
    received_packets_.push_back(std::move(received_packet));
  }
  return true;
}

  3.AddReceivedRedPacket函数处理掉RED头，然后将解RED后的包放入received_packets_队列中。

int32_t UlpfecReceiverImpl::ProcessReceivedFec() {
  crit_sect_.Enter();

  // If we iterate over |received_packets_| and it contains a packet that cause
  // us to recurse back to this function (for example a RED packet encapsulating
  // a RED packet), then we will recurse forever. To avoid this we swap
  // |received_packets_| with an empty vector so that the next recursive call
  // wont iterate over the same packet again. This also solves the problem of
  // not modifying the vector we are currently iterating over (packets are added
  // in AddReceivedRedPacket).
  std::vector<std::unique_ptr<ForwardErrorCorrection::ReceivedPacket>>
      received_packets;
  received_packets.swap(received_packets_);
  // RTC_LOG(LS_ERROR) << "kiki-fec/size " << received_packets.size();

  for (const auto& received_packet : received_packets) {
    // Send received media packet to VCM.
    if (!received_packet->is_fec) {
      ForwardErrorCorrection::Packet* packet = received_packet->pkt;
      crit_sect_.Leave();
      recovered_packet_callback_->OnRecoveredPacket(packet->data.data(),
                                                    packet->data.size());
      crit_sect_.Enter();
      // Create a packet with the buffer to modify it.
      RtpPacketReceived rtp_packet;
      const uint8_t* const original_data = packet->data.cdata();
      if (!rtp_packet.Parse(packet->data)) {
        RTC_LOG(LS_WARNING) << "Corrupted media packet";
      } else {
        rtp_packet.IdentifyExtensions(extensions_);
        // Reset buffer reference, so zeroing would work on a buffer with a
        // single reference.
        packet->data = rtc::CopyOnWriteBuffer(0);
        rtp_packet.ZeroMutableExtensions();
        packet->data = rtp_packet.Buffer();
        // Ensure that zeroing of extensions was done in place.
        RTC_DCHECK_EQ(packet->data.cdata(), original_data);
      }
    }
    if (!received_packet->is_recovered) {
      // Do not pass recovered packets to FEC. Recovered packet might have
      // different set of the RTP header extensions and thus different byte
      // representation than the original packet, That will corrupt
      // FEC calculation.
      fec_->DecodeFec(*received_packet, &recovered_packets_);
    }
  }

  // Send any recovered media packets to VCM.
  for (const auto& recovered_packet : recovered_packets_) {
    // RTC_LOG(LS_ERROR) << "kiki-fec/recovered: " << recovered_packet->seq_num;
    if (recovered_packet->returned) {
      // Already sent to the VCM and the jitter buffer.
      continue;
    }
    ForwardErrorCorrection::Packet* packet = recovered_packet->pkt;
    ++packet_counter_.num_recovered_packets;
    // Set this flag first; in case the recovered packet carries a RED
    // header, OnRecoveredPacket will recurse back here.
    recovered_packet->returned = true;
    crit_sect_.Leave();
    recovered_packet_callback_->OnRecoveredPacket(packet->data.data(),
                                                  packet->data.size());
    crit_sect_.Enter();
  }

  crit_sect_.Leave();
  return 0;
}

  4.ProcessReceivedFec首先对每个处于received_packets_队列中的包进行处理（从代码来看，因为received_packets_只有在前述AddReceivedRedPacket会做插入操作，而前述函数每次只插入一个包，故这里实际每次只处理一个包）。对于视频包，会先调第二次的ReceivePacket，并最终调用OnReceivedPayloadData，以保证视频帧的尽快渲染。接着对于所有包（无论视频包还是fec包的is_recovered都是false），会做前述fec处理逻辑，尝试恢复丢包。最后循环recovered_packets_队列，如果有returned为false的包，说明是被fec恢复的丢包，如普通视频包一样调用第二次ReceivePacket。

void RtpVideoStreamReceiver::OnRecoveredPacket(const uint8_t* rtp_packet,
                                               size_t rtp_packet_length) {
  RtpPacketReceived packet;
  if (!packet.Parse(rtp_packet, rtp_packet_length))
    return;
  if (packet.PayloadType() == config_.rtp.red_payload_type) {
    RTC_LOG(LS_WARNING) << "Discarding recovered packet with RED encapsulation";
    return;
  }

  packet.IdentifyExtensions(rtp_header_extensions_);
  packet.set_payload_type_frequency(kVideoPayloadTypeFrequency);
  // TODO(nisse): UlpfecReceiverImpl::ProcessReceivedFec passes both
  // original (decapsulated) media packets and recovered packets to
  // this callback. We need a way to distinguish, for setting
  // packet.recovered() correctly. Ideally, move RED decapsulation out
  // of the Ulpfec implementation.

  ReceivePacket(packet);
}

  5.如代码注释所写，包的recover标志位并没有被正确设置，即使是被fec恢复的包，is_recovered依然是false。