webrtc rtp解码流程分析

版权申明：未经允许请勿转载。转载前请先联系作者（hello@yeshen.org）

最近在研究webrtc的视频编码的东西。是解码的时候出现了问题，带着问题，现在研究下WEBRTC H264的编解码流程。

代码分析

代码入口在这里-> rtp_video_stream_receiver.cc

1, RTP的数据底层获取之后，传递给 RtpVideoStreamReceiver::OnRecoveredPacket

rtp_video_stream_receiver.cc

void RtpVideoStreamReceiver::OnRecoveredPacket(const uint8_t* rtp_packet,
                                               size_t rtp_packet_length) {
  ...
  ReceivePacket(packet);
}

void RtpVideoStreamReceiver::ReceivePacket(const RtpPacketReceived& packet) {
  ...
  RtpDepacketizer::ParsedPayload parsed_payload;
  if (!depacketizer->Parse(&parsed_payload, packet.payload().data(),
                           packet.payload().size())) {
    RTC_LOG(LS_WARNING) << "Failed parsing payload.";
    return;
  }

  WebRtcRTPHeader webrtc_rtp_header = {};
  packet.GetHeader(&webrtc_rtp_header.header);

  webrtc_rtp_header.frameType = parsed_payload.frame_type;
  webrtc_rtp_header.video_header() = parsed_payload.video_header();
  webrtc_rtp_header.video_header().rotation = kVideoRotation_0;
  webrtc_rtp_header.video_header().content_type = VideoContentType::UNSPECIFIED;
  webrtc_rtp_header.video_header().video_timing.flags =
      VideoSendTiming::kInvalid;
  webrtc_rtp_header.video_header().playout_delay.min_ms = -1;
  webrtc_rtp_header.video_header().playout_delay.max_ms = -1;

  // Retrieve the video rotation information.
  packet.GetExtension<VideoOrientation>(
      &webrtc_rtp_header.video_header().rotation);

  packet.GetExtension<VideoContentTypeExtension>(
      &webrtc_rtp_header.video_header().content_type);
  packet.GetExtension<VideoTimingExtension>(
      &webrtc_rtp_header.video_header().video_timing);
  packet.GetExtension<PlayoutDelayLimits>(
      &webrtc_rtp_header.video_header().playout_delay);

  OnReceivedPayloadData(parsed_payload.payload, parsed_payload.payload_length,
                        &webrtc_rtp_header);
}

2，其中，depacketizer->Parse 会选择特定编码方式的解包，比如rtp_format_h264.cc
3， OnReceivedPayloadData 处理完数据包之后，插入 packet_buffer_->InsertPacket(&packet);

rtp_video_stream_receiver.cc

int32_t RtpVideoStreamReceiver::OnReceivedPayloadData(
    const uint8_t* payload_data,
    size_t payload_size,
    const WebRtcRTPHeader* rtp_header) {
  ...
  packet_buffer_->InsertPacket(&packet);
  return 0;
}

4, InsertPacket 之后判断收到的数据帧是否完整，完整的话，就提交给上层做解码。

rtp_video_stream_receiver.cc

void RtpVideoStreamReceiver::OnReceivedFrame(
    std::unique_ptr<video_coding::RtpFrameObject> frame) {
  if (!has_received_frame_) {
    has_received_frame_ = true;
    if (frame->FrameType() != kVideoFrameKey)
      keyframe_request_sender_->RequestKeyFrame();
  }
  RTC_LOG(LS_ERROR) << "Yeshen : OnReceivedFrame.";

  reference_finder_->ManageFrame(std::move(frame));
}

5，具体的判断逻辑如下：

rtp_frame_reference_finder.cc

void RtpFrameReferenceFinder::ManageFrame(
    std::unique_ptr<RtpFrameObject> frame) {
  rtc::CritScope lock(&crit_);

  // If we have cleared past this frame, drop it.
  if (cleared_to_seq_num_ != -1 &&
      AheadOf<uint16_t>(cleared_to_seq_num_, frame->first_seq_num())) {
    return;
  }

  FrameDecision decision = ManageFrameInternal(frame.get());
  switch (decision) {
    case kStash:
      if (stashed_frames_.size() > kMaxStashedFrames)
        stashed_frames_.pop_back();
      stashed_frames_.push_front(std::move(frame));
      RTC_LOG(LS_ERROR) << "Yeshen : ManageFrameInternal. kStash";
      break;
    case kHandOff:
      frame_callback_->OnCompleteFrame(std::move(frame));
      RetryStashedFrames();
      RTC_LOG(LS_ERROR) << "Yeshen : ManageFrameInternal. kHandOff";
      break;
    case kDrop:
      RTC_LOG(LS_ERROR) << "Yeshen : ManageFrameInternal. kDrop";
      break;
  }
}

return kHandOff 说明数据是完整的

rtp_frame_reference_finder.cc

RtpFrameReferenceFinder::FrameDecision
RtpFrameReferenceFinder::ManageFrameGeneric(RtpFrameObject* frame,
                                            int picture_id) {
  // If |picture_id| is specified then we use that to set the frame references,
  // otherwise we use sequence number.
  if (picture_id != kNoPictureId) {
    if (last_unwrap_ == -1)
      last_unwrap_ = picture_id;

    frame->id.picture_id = unwrapper_.Unwrap(picture_id);
    frame->num_references = frame->frame_type() == kVideoFrameKey ? 0 : 1;
    frame->references[0] = frame->id.picture_id - 1;
    return kHandOff;
  }

  if (frame->frame_type() == kVideoFrameKey) {
    last_seq_num_gop_.insert(std::make_pair(
        frame->last_seq_num(),
        std::make_pair(frame->last_seq_num(), frame->last_seq_num())));
  }

  // We have received a frame but not yet a keyframe, stash this frame.
  if (last_seq_num_gop_.empty())
    return kStash;

  // Clean up info for old keyframes but make sure to keep info
  // for the last keyframe.
  auto clean_to = last_seq_num_gop_.lower_bound(frame->last_seq_num() - 100);
  for (auto it = last_seq_num_gop_.begin();
       it != clean_to && last_seq_num_gop_.size() > 1;) {
    it = last_seq_num_gop_.erase(it);
  }

  // Find the last sequence number of the last frame for the keyframe
  // that this frame indirectly references.
  auto seq_num_it = last_seq_num_gop_.upper_bound(frame->last_seq_num());
  if (seq_num_it == last_seq_num_gop_.begin()) {
    RTC_LOG(LS_WARNING) << "Generic frame with packet range ["
                        << frame->first_seq_num() << ", "
                        << frame->last_seq_num()
                        << "] has no GoP, dropping frame.";
    return kDrop;
  }
  seq_num_it--;

  // Make sure the packet sequence numbers are continuous, otherwise stash
  // this frame.
  uint16_t last_picture_id_gop = seq_num_it->second.first;
  uint16_t last_picture_id_with_padding_gop = seq_num_it->second.second;
  if (frame->frame_type() == kVideoFrameDelta) {
    uint16_t prev_seq_num = frame->first_seq_num() - 1;

    if (prev_seq_num != last_picture_id_with_padding_gop)
      return kStash;
  }

  RTC_DCHECK(AheadOrAt(frame->last_seq_num(), seq_num_it->first));

  // Since keyframes can cause reordering we can't simply assign the
  // picture id according to some incrementing counter.
  frame->id.picture_id = frame->last_seq_num();
  frame->num_references = frame->frame_type() == kVideoFrameDelta;
  frame->references[0] = generic_unwrapper_.Unwrap(last_picture_id_gop);
  if (AheadOf<uint16_t>(frame->id.picture_id, last_picture_id_gop)) {
    seq_num_it->second.first = frame->id.picture_id;
    seq_num_it->second.second = frame->id.picture_id;
  }

  last_picture_id_ = frame->id.picture_id;
  UpdateLastPictureIdWithPadding(frame->id.picture_id);
  frame->id.picture_id = generic_unwrapper_.Unwrap(frame->id.picture_id);
  return kHandOff;
}

未完待续

参考资料：
1, https://github.com/jitsi/webrtc/blob/M75/video/rtp_video_stream_receiver.cc
2, https://www.jianshu.com/p/9f7cfc185e6e