webrtc周期性发送RTPFB反馈包

   本文基于m79版本的webrtc代码走读，其他版本代码可能有所不同，但是原理没有太大变化。接收端接收到远端的音视频数据包后，会周期的发送RTPFB反馈包记录接收时间和发送时间，用于发送端基于延时的码率估计，周期时间一般为100ms。socket接受到数据包并解析包后，通过带宽预测模块记录接收的包信息，周期发送rtpfb模块构造自上一次发送rtpfb包到本次构造rtpfb包之间的时间段内的包信息。
一、RTPFB包的结构
   rtpfb包的结构如下所示：

 0               1               2               3
   0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7
  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  |V=2|P|  FMT=15 |    PT=205     |           length              |
  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  |                     SSRC of packet sender                     |
  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  |                      SSRC of media source                     |
  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  |      base sequence number     |      packet status count      |
  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  |                 reference time                | fb pkt. count |
  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  |          packet chunk         |         packet chunk          |
  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  .                                                               .
  .                                                               .
  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  |         packet chunk          |  recv delta   |  recv delta   |
  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  .                                                               .
  .                                                               .
  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  |           recv delta          |  recv delta   | zero padding  |
  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 */

FMT对应值如下：
            NACK = 1,
            TMMBR = 3,
            TMMBN = 4,
            SR_REQ = 5,
            RAMS = 6,
            TLLEI = 7,
            ECN = 8,
            PS = 9,
            TCC = 15,
            EXT = 31
SSRC of packet sender : 表示RTFB发送包所在所对应的流ssrc，ssrc=0，不单独创建一个会话来发送RTFB。
SSRC of media source ：表示RTFB接收方对应的流ssrc，不是所有的流都可以用这个RTFB的，音频1是音频1接收，音频2是音频2接收，视频是视频接收，互不影响
base sequence number：基础序列号，上一个RTPFB包发送后接收的第一个包的序列号+1，作为下一个RTPFB包中的基础序列号；
packet status count ：接收数据包状态产生的状态包计数；
reference time ：参考时间，为构造当前rtfp时第一个数据包的到达时间；
fb pkt. count：过往已经发送的RTPFB个数；
packet chunk：状态包；
recv delta：接收相邻包的时间间隔；
zero padding：填充0，确保4字节对齐；
二、接收音视频数据包并保存
   接受到音视频数据包之后，传入接收端拥塞控制模块的远程估计RemoteEstimatorProxy中记录序列号对应的到达时间map组packet_arrival_times_，代码走读如下：

void RemoteEstimatorProxy::IncomingPacket(int64_t arrival_time_ms,
                                          size_t payload_size,
                                          const RTPHeader& header) {
  if (arrival_time_ms < 0 || arrival_time_ms > kMaxTimeMs) {
    RTC_LOG(LS_WARNING) << "Arrival time out of bounds: " << arrival_time_ms;
    return;
  }
  rtc::CritScope cs(&lock_);
  media_ssrc_ = header.ssrc;
  int64_t seq = 0;

  if (header.extension.hasTransportSequenceNumber) {
    seq = unwrapper_.Unwrap(header.extension.transportSequenceNumber);//解析包的序列号

    if (send_periodic_feedback_) {
    //periodic_window_start_seq_为本次rtpfb包中的base sequence number，periodic_window_start_seq_初始化为空值，收到第一个包时赋值。
    //如果packet_arrival_times_的map中没有记录接收的req时，即不是重传的包，从map第一个元素开始检测是否超过500ms的范围，如果第一个包和
    //收到的包的到达时间超过500ms，则删除，依次后续的包比较，目的就是移除旧的已处理的包。
      if (periodic_window_start_seq_ &&
          packet_arrival_times_.lower_bound(*periodic_window_start_seq_) ==
              packet_arrival_times_.end()) {
        // Start new feedback packet, cull old packets.
        for (auto it = packet_arrival_times_.begin();
             it != packet_arrival_times_.end() && it->first < seq &&
             arrival_time_ms - it->second >= send_config_.back_window->ms();) {
          it = packet_arrival_times_.erase(it);
        }
      }
      if (!periodic_window_start_seq_ || seq < *periodic_window_start_seq_) {
        periodic_window_start_seq_ = seq;//1. 收到启动后的第一个包时将其序列号赋值
        //2. seq收到的序列号小于periodic_window_start_seq_时，重新赋予新值seq
      }
    }

    // 检测.packet_arrival_times_中是否已存在，如果存在，不再插入
    if (packet_arrival_times_.find(seq) != packet_arrival_times_.end())
      return;

    packet_arrival_times_[seq] = arrival_time_ms;//插入到packet_arrival_times_，

    // 限制发送rtpfb的序列号范围
    auto first_arrival_time_to_keep = packet_arrival_times_.lower_bound(
        packet_arrival_times_.rbegin()->first - kMaxNumberOfPackets);
    if (first_arrival_time_to_keep != packet_arrival_times_.begin()) {
      packet_arrival_times_.erase(packet_arrival_times_.begin(),
                                  first_arrival_time_to_keep);
      if (send_periodic_feedback_) {
        // |packet_arrival_times_| cannot be empty since we just added one
        // element and the last element is not deleted.
        RTC_DCHECK(!packet_arrival_times_.empty());
        periodic_window_start_seq_ = packet_arrival_times_.begin()->first;
      }
    }
     //如果发送端设置feedback_request=true,表明接收端应立即发送rtpfb，而不需要等100ms到来
    if (header.extension.feedback_request) {
      // Send feedback packet immediately.
      SendFeedbackOnRequest(seq, header.extension.feedback_request.value());
    }
  }
//如果需要对网络状态进行估计且数据包携带绝对发送时间，则需要将接受包传入network_state_estimator_模块记录保存。
//webrtc不对网络状态进行估计，所有的都在网络拥塞控制和码率控制中进行
  if (network_state_estimator_ && header.extension.hasAbsoluteSendTime) {
    PacketResult packet_result;
    packet_result.receive_time = Timestamp::ms(arrival_time_ms);
    // Ignore reordering of packets and assume they have approximately the same
    // send time.
    abs_send_timestamp_ += std::max(
        header.extension.GetAbsoluteSendTimeDelta(previous_abs_send_time_),
        TimeDelta::ms(0));
    previous_abs_send_time_ = header.extension.absoluteSendTime;
    packet_result.sent_packet.send_time = abs_send_timestamp_;
    // TODO(webrtc:10742): Take IP header and transport overhead into account.
    packet_result.sent_packet.size =
        DataSize::bytes(header.headerLength + payload_size);
    packet_result.sent_packet.sequence_number = seq;
    network_state_estimator_->OnReceivedPacket(packet_result);
  }
}

三、RTPFB的packet chunk和delta
packet chunk有三种类型：决策编码类型代码：

uint16_t TransportFeedback::LastChunk::EncodeLast() const {
  RTC_DCHECK_GT(size_, 0);
  if (all_same_)
    return EncodeRunLength();
  if (size_ <= kMaxTwoBitCapacity)
    return EncodeTwoBit(size_);
  return EncodeOneBit();
}

3.1、packet chunk类型编解码——RunLenghtChunk

   所有包状态都一致(all_same_=true)时编码该类型packet chunk，chunk的两个字节定义如下图：

chunk编码：

uint16_t TransportFeedback::LastChunk::EncodeRunLength() const {
  RTC_DCHECK(all_same_);
  RTC_DCHECK_LE(size_, kMaxRunLengthCapacity);
  //delta_sizes_保存了每个包与前一个包的到达时间差的字节数：0或1或2，因为是EncodeRunLength，所有delta状态
  //一致，因此取第一个delta_sizes_[0]
  //size_为chunk中多少个delta
  return (delta_sizes_[0] << 13) | static_cast<uint16_t>(size_);
}

chunk解码：

void TransportFeedback::LastChunk::DecodeRunLength(uint16_t chunk,
                                                   size_t max_count) {
  RTC_DCHECK_EQ(chunk & 0x8000, 0);
  size_ = std::min<size_t>(chunk & 0x1fff, max_count);
  DeltaSize delta_size = (chunk >> 13) & 0x03;
  has_large_delta_ = delta_size >= kLarge;
  all_same_ = true;
  // To make it consistent with Add function, populate delta_sizes_ beyound 1st.
  for (size_t i = 0; i < std::min<size_t>(size_, kMaxVectorCapacity); ++i)
    delta_sizes_[i] = delta_size;
}

3.2、packet chunk类型——TwoBitVectorChunks
   如果状态delta个数小于等于7个时编码该类型packet chunk，最高两位为1，其次每个包的delta字节占用两位，chunk的两个字节定义如下图：
chunk编码：

uint16_t TransportFeedback::LastChunk::EncodeTwoBit(size_t size) const {
  RTC_DCHECK_LE(size, size_);
  uint16_t chunk = 0xc000;
  //kMaxTwoBitCapacity=7
  for (size_t i = 0; i < size; ++i)
    chunk |= delta_sizes_[i] << 2 * (kMaxTwoBitCapacity - 1 - i);
  return chunk;
}

chunk解码：

void TransportFeedback::LastChunk::DecodeTwoBit(uint16_t chunk,
                                                size_t max_size) {
  RTC_DCHECK_EQ(chunk & 0xc000, 0xc000);
  size_ = std::min(kMaxTwoBitCapacity, max_size);
  has_large_delta_ = true;
  all_same_ = false;
  for (size_t i = 0; i < size_; ++i)
    delta_sizes_[i] = (chunk >> 2 * (kMaxTwoBitCapacity - 1 - i)) & 0x03;
}

3.3、packet chunk类型——OneBitVectorChunks
OneBitVectorChunks:其他情况，这种chunk的delta_size=1或0，不可能为2，chunk的两个字节定义如下图：
类型决策代码如下：

chunk编码：

uint16_t TransportFeedback::LastChunk::EncodeOneBit() const {
  RTC_DCHECK(!has_large_delta_);
  RTC_DCHECK_LE(size_, kMaxOneBitCapacity);
  uint16_t chunk = 0x8000;
  for (size_t i = 0; i < size_; ++i)
    chunk |= delta_sizes_[i] << (kMaxOneBitCapacity - 1 - i);
  return chunk;
}

chunk解码：

void TransportFeedback::LastChunk::DecodeOneBit(uint16_t chunk,
                                                size_t max_size) {
  RTC_DCHECK_EQ(chunk & 0xc000, 0x8000);
  size_ = std::min(kMaxOneBitCapacity, max_size);
  has_large_delta_ = false;
  all_same_ = false;
  for (size_t i = 0; i < size_; ++i)
    delta_sizes_[i] = (chunk >> (kMaxOneBitCapacity - 1 - i)) & 0x01;
}

3.4、delta
   rtpfb中记录当前窗口内所有相邻包的到达时间差，在packet chunk之后记录delta，只有知道前一个到达时间last_arriver_time和delta，就可以计算出来当前包的到达时间cur_arriver_time=last_arriver_time+delta;

//received_packets_存储接收包的系列号seq和delta值，依次将delta写入到rtpfb中
for (const auto& received_packet : received_packets_) {
      int16_t delta = received_packet.delta_ticks();
      if (delta >= 0 && delta <= 0xFF) {
        packet[(*position)++] = delta;//一个字节的delta
      } else {//两个字节的delta
        ByteWriter<int16_t>::WriteBigEndian(&packet[*position], delta);
        *position += 2;
      }
    }

四、周期发送
   接收端RemoteEstimatorProxy会开启线程定时100ms发送一次rtpfb包，如果是多人会议系统可以根据参会人修改定时时间，执行函数SendPeriodicFeedbacks，代码走读解释如下：

void RemoteEstimatorProxy::SendPeriodicFeedbacks() {
  // |periodic_window_start_seq_| 是当前rtpfb包中的第一个包序列号，如果不存在表示还没有接受到第一个包，不会发送rtpfb
  if (!periodic_window_start_seq_)
    return;
  for (auto begin_iterator =
           packet_arrival_times_.lower_bound(*periodic_window_start_seq_);
       begin_iterator != packet_arrival_times_.cend();
       begin_iterator =
           packet_arrival_times_.lower_bound(*periodic_window_start_seq_)) {
    auto feedback_packet = std::make_unique<rtcp::TransportFeedback>();
    periodic_window_start_seq_ = BuildFeedbackPacket(//在packet_arrival_times_中从起始seq=periodic_window_start_seq_到最后一个数据包
    //的所有接收包构造rtpfb反馈包
        feedback_packet_count_++, media_ssrc_, *periodic_window_start_seq_,
        begin_iterator, packet_arrival_times_.cend(), feedback_packet.get());

    RTC_DCHECK(feedback_sender_ != nullptr);

    std::vector<std::unique_ptr<rtcp::RtcpPacket>> packets;
    if (remote_estimate) {
      packets.push_back(std::move(remote_estimate));
    }
    packets.push_back(std::move(feedback_packet));//可能一次存在多个反馈包

    feedback_sender_->SendCombinedRtcpPacket(std::move(packets));//发送构造的rtpfb包
    //此处需要注意：发送完rtpfb后，不会移除packet_arrival_times_ map中已构造过rtpfb的包，这是以防重排序后需要重传，移除
    //的操作是在插入新包到packet_arrival_times_ 时，超过500ms窗口的旧包
  }
}

   比较重要的是如何构造rtpfb反馈包，由函数BuildFeedbackPacket完成，代码如下：

int64_t RemoteEstimatorProxy::BuildFeedbackPacket(
    uint8_t feedback_packet_count,//
    uint32_t media_ssrc,
    int64_t base_sequence_number,
    std::map<int64_t, int64_t>::const_iterator begin_iterator,
    std::map<int64_t, int64_t>::const_iterator end_iterator,
    rtcp::TransportFeedback* feedback_packet) {
  RTC_DCHECK(begin_iterator != end_iterator);

  //设置rtfpb结构中的SSRC of media source
  feedback_packet->SetMediaSsrc(media_ssrc);
  //设置rtfpb结构中的base sequence number和reference time 
  feedback_packet->SetBase(static_cast<uint16_t>(base_sequence_number & 0xFFFF),
                           begin_iterator->second * 1000);
  //设置rtfpb结构中的*fb pkt.count
  feedback_packet->SetFeedbackSequenceNumber(feedback_packet_count);
  int64_t next_sequence_number = base_sequence_number;
  //构造状态包，传入序列号和时间戳，需要计算包与包之间的接收时间差
  for (auto it = begin_iterator; it != end_iterator; ++it) {
    if (!feedback_packet->AddReceivedPacket(
            static_cast<uint16_t>(it->first & 0xFFFF), it->second * 1000)) {
      // If we can't even add the first seq to the feedback packet, we won't be
      // able to build it at all.
      RTC_CHECK(begin_iterator != it);

      // Could not add timestamp, feedback packet might be full. Return and
      // try again with a fresh packet.
      break;
    }
    next_sequence_number = it->first + 1;
  }
  return next_sequence_number;
}

构造rtpfb中的状态包代码实现如下：

bool TransportFeedback::AddReceivedPacket(uint16_t sequence_number,
                                          int64_t timestamp_us) {
  int16_t delta = 0;
  if (include_timestamps_) {
    // Convert to ticks and round.
    int64_t delta_full =
        (timestamp_us - last_timestamp_us_) % kTimeWrapPeriodUs;//计算包与上一次包接收实际差并乘以一个250us的因子
    if (delta_full > kTimeWrapPeriodUs / 2)
      delta_full -= kTimeWrapPeriodUs;
    delta_full +=
        delta_full < 0 ? -(kDeltaScaleFactor / 2) : kDeltaScaleFactor / 2;
    delta_full /= kDeltaScaleFactor;

    delta = static_cast<int16_t>(delta_full);//delta最大两个字节，如果超过两个字节，不会构造rtpfb
    if (delta != delta_full) {
      RTC_LOG(LS_WARNING) << "Delta value too large ( >= 2^16 ticks )";
      return false;
    }
  }

  uint16_t next_seq_no = base_seq_no_ + num_seq_no_;
  if (sequence_number != next_seq_no) {//如果丢失了序列号sequence_number的包，则添加delta=0到rtpfb中
    uint16_t last_seq_no = next_seq_no - 1;
    if (!IsNewerSequenceNumber(sequence_number, last_seq_no))
      return false;
    for (; next_seq_no != sequence_number; ++next_seq_no)
      if (!AddDeltaSize(0))丢失的包添加delta=0到rtpfb中
        return false;
  }

  DeltaSize delta_size = (delta >= 0 && delta <= 0xff) ? 1 : 2;
  if (!AddDeltaSize(delta_size))//非丢失的包添加delta到rtpfb中
    return false;

  received_packets_.emplace_back(sequence_number, delta);
  last_timestamp_us_ += delta * kDeltaScaleFactor;
  if (include_timestamps_) {
    size_bytes_ += delta_size;
  }
  return true;
}

从rtpfb包的结构中能看出可以包含很多packet chunk和delta，packet chunk和delta具有对应关系。一个packet chunk中最少要包含7个delta(如果在100ms期间内接受到的包数量不足7个，那么rtfp中只包含一个packet chunk)。
rtpfb中包含多少packet chunk的决策：

1. 如果本次rtpfb构造时间内接收的包个数小于7个，一定只有一个packet chunk；
2. 如果本次rtpfb构造时间内接收的包数大于7个且小于14个，如果存在delta=2字节的情况时，一定有两个packet chunk，如果不存在delta=2    字节的情况时，一定也只有一个packet chunk；
3. 如果本次接收的包不少于14个包，先判断1和2的情况，如果1和2都满足一个packet chunk的情况，再判断是否状态一致，如果一致也   只有一个packet chunk，不一致的话也会存在多于一个的packet chunk。这里的状态是指包与包之间的到达时间差delta都为一个字节或都为两个字节。
   代码如下：

bool TransportFeedback::LastChunk::CanAdd(DeltaSize delta_size) const {
  RTC_DCHECK_LE(delta_size, 2);
  if (size_ < kMaxTwoBitCapacity)//kMaxTwoBitCapacity=7
    return true;
  if (size_ < kMaxOneBitCapacity && !has_large_delta_ && delta_size != kLarge)//kMaxOneBitCapacity =14
    return true;
  if (size_ < kMaxRunLengthCapacity && all_same_ &&
      delta_sizes_[0] == delta_size)
    return true;
  return false;
}

bool TransportFeedback::AddDeltaSize(DeltaSize delta_size) {
  if (num_seq_no_ == kMaxReportedPackets)
    return false;
  size_t add_chunk_size = last_chunk_.Empty() ? kChunkSizeBytes : 0;
  if (size_bytes_ + delta_size + add_chunk_size > kMaxSizeBytes)
    return false;

  if (last_chunk_.CanAdd(delta_size)) {//判断packet chunk数量，如果返回成功，packet chunk=1，返回失败packet chunk>1
    size_bytes_ += add_chunk_size;
    last_chunk_.Add(delta_size);
    ++num_seq_no_;
    return true;
  }
  if (size_bytes_ + delta_size + kChunkSizeBytes > kMaxSizeBytes)
    return false;

  encoded_chunks_.push_back(last_chunk_.Emit());//只有last_chunk_.CanAdd返回失败，packet chunk>1
  size_bytes_ += kChunkSizeBytes;
  last_chunk_.Add(delta_size);
  ++num_seq_no_;
  return true;
}