三问 WebRTC 视频编码分辨率

众所周知，WebRTC 自带带宽预测功能，可以根据设置的视频降级策略，选择保流畅（MAINTAIN_FRAMERATE）还是保清晰（MAINTAIN_RESOLUTION）或是均衡策略(BALANCED)。故此，有以下三问：

1. 视频编码的最大分辨率如何限制？

2. 视频编码的起始分辨率如何确定？

3. 视频编码的最小分辨率如何保证？

问题 1： 视频编码的最大分辨率如何限制？

WebRTC内视频 VideoSource（AdaptedVideoTrackSource）负责对采集帧进行初步处理，例如丢帧，裁剪等，主要处理都在 VideoAdapter。而ObjCVideoTrackSource (iOS) 和 AndroidVideoTrackSource 继承于 AdaptedVideoTrackSource。以 iOS 为例，设置输出分辨率。

• 分辨率及帧率设置流程：

上层调用 adaptOutputFormatToWidth() 接口，传入期望的分辨率宽高及帧率。

// sdk/objc/api/peerconnection/RTCVideoSource.mm
- (void)adaptOutputFormatToWidth:(int)width height:(int)height fps:(int)fps {
  getObjCVideoSource(_nativeVideoSource)->OnOutputFormatRequest(width, height, fps);
}

// sdk/objc/native/src/objc_video_trace_source.mm
void ObjCVideoTrackSource::OnOutputFormatRequest(int width, int height, int fps) {
  cricket::VideoFormat format(width, height, cricket::VideoFormat::FpsToInterval(fps), 0);
  video_adapter()->OnOutputFormatRequest(format);
}

VideoAdapter 处理视频采集格式输出请求。

void VideoAdapter::OnOutputFormatRequest(
    const absl::optional<VideoFormat>& format) {
  absl::optional<std::pair<int, int>> target_aspect_ratio;
  absl::optional<int> max_pixel_count;
  absl::optional<int> max_fps;
  if (format) {
    // 分辨率比例
    target_aspect_ratio = std::make_pair(format->width, format->height);
    // 最大分辨率像素数
    max_pixel_count = format->width * format->height;
    if (format->interval > 0)
      // 最大帧率
      max_fps = rtc::kNumNanosecsPerSec / format->interval;
  }
  OnOutputFormatRequest(target_aspect_ratio, max_pixel_count, max_fps);
}

void VideoAdapter::OnOutputFormatRequest(
    const absl::optional<std::pair<int, int>>& target_aspect_ratio,
    const absl::optional<int>& max_pixel_count,
    const absl::optional<int>& max_fps) {
  absl::optional<std::pair<int, int>> target_landscape_aspect_ratio;
  absl::optional<std::pair<int, int>> target_portrait_aspect_ratio;
  if (target_aspect_ratio && target_aspect_ratio->first > 0 &&
      target_aspect_ratio->second > 0) {
    // Maintain input orientation.
    // 转换横竖屏分辨率宽高比例
    const int max_side =
        std::max(target_aspect_ratio->first, target_aspect_ratio->second);
    const int min_side =
        std::min(target_aspect_ratio->first, target_aspect_ratio->second);
    target_landscape_aspect_ratio = std::make_pair(max_side, min_side);
    target_portrait_aspect_ratio = std::make_pair(min_side, max_side);
  }
  OnOutputFormatRequest(target_landscape_aspect_ratio, max_pixel_count,
                        target_portrait_aspect_ratio, max_pixel_count, max_fps);
}

void VideoAdapter::OnOutputFormatRequest(
    const absl::optional<std::pair<int, int>>& target_landscape_aspect_ratio,
    const absl::optional<int>& max_landscape_pixel_count,
    const absl::optional<std::pair<int, int>>& target_portrait_aspect_ratio,
    const absl::optional<int>& max_portrait_pixel_count,
    const absl::optional<int>& max_fps) {
  webrtc::MutexLock lock(&mutex_);
  // 横屏时分辨率宽高比例
  target_landscape_aspect_ratio_ = target_landscape_aspect_ratio;
  // 横屏最大分辨率像素数
  max_landscape_pixel_count_ = max_landscape_pixel_count;
  // 竖屏时分辨率宽高比
  target_portrait_aspect_ratio_ = target_portrait_aspect_ratio;
  // 竖屏最大分辨率像素数
  max_portrait_pixel_count_ = max_portrait_pixel_count;
  // 最大帧率
  max_fps_ = max_fps;
  next_frame_timestamp_ns_ = absl::nullopt;
}

当视频编码流的Sink添加到 Source 时，调用 AdaptedVideoTrackSource::AddOrUpdateSink(sink, wants)

// media/base/adapted_video_track_source.cc
void AdaptedVideoTrackSource::AddOrUpdateSink(
    rtc::VideoSinkInterface<webrtc::VideoFrame>* sink,
    const rtc::VideoSinkWants& wants) {
  // sink 管理台更新 sink，内部会调用 UpdateWants()
  broadcaster_.AddOrUpdateSink(sink, wants);
  // sink 变化之后，wants 也会跟着变化，broadcaster_wants() 返回 current_wants_，在UpdateWants()里面已更新
  OnSinkWantsChanged(broadcaster_.wants());
}

void AdaptedVideoTrackSource::OnSinkWantsChanged(
    const rtc::VideoSinkWants& wants) {
  video_adapter_.OnSinkWants(wants);
}

VideoBroadcaster 负责分发视频帧到各个sink，故需保存 sink和wants，统一各sink的 wants 的最大分辨率像素数，目标分辨率像素数及最大帧率。

// media/base/video_broadcaster.cc
void VideoBroadcaster::AddOrUpdateSink(
    VideoSinkInterface<webrtc::VideoFrame>* sink,
    const VideoSinkWants& wants) {
  RTC_DCHECK(sink != nullptr);
  webrtc::MutexLock lock(&sinks_and_wants_lock_);
  if (!FindSinkPair(sink)) {
    // |Sink| is a new sink, which didn't receive previous frame.
    previous_frame_sent_to_all_sinks_ = false;
  }
  // 保存sink 和 wants 的配对
  VideoSourceBase::AddOrUpdateSink(sink, wants);
  // 更新 wants，之后给到 VideoAdapter
  UpdateWants();
}

void VideoBroadcaster::UpdateWants() {
  VideoSinkWants wants;
  wants.rotation_applied = false;
  wants.resolution_alignment = 1;
  // 获取所有sink都能适应的wants，即所有 sink里面的最小 wants 值
  for (auto& sink : sink_pairs()) {
    // wants.rotation_applied == ANY(sink.wants.rotation_applied)
    if (sink.wants.rotation_applied) {
      wants.rotation_applied = true;
    }
    // wants.max_pixel_count == MIN(sink.wants.max_pixel_count)
    // 获取所有sink 的 wants 里面 max_pixel_count 的最小值
    if (sink.wants.max_pixel_count < wants.max_pixel_count) {
      wants.max_pixel_count = sink.wants.max_pixel_count;
    }
    // Select the minimum requested target_pixel_count, if any, of all sinks so
    // that we don't over utilize the resources for any one.
    // TODO(sprang): Consider using the median instead, since the limit can be
    // expressed by max_pixel_count.
    if (sink.wants.target_pixel_count &&
        (!wants.target_pixel_count ||
         (*sink.wants.target_pixel_count < *wants.target_pixel_count))) {
      // 获取所有 sink的 wants 的最小 target_pixel_count 值
      wants.target_pixel_count = sink.wants.target_pixel_count;
    }
    // Select the minimum for the requested max framerates.
    if (sink.wants.max_framerate_fps < wants.max_framerate_fps) {
      // 获取所有sink 的wants的最小 max_framerate_fps
      wants.max_framerate_fps = sink.wants.max_framerate_fps;
    }
    wants.resolution_alignment = cricket::LeastCommonMultiple(
        wants.resolution_alignment, sink.wants.resolution_alignment);
  }

  // 如果 target_pixel_count > max_pixel_count，修正 target_pixel_count 为 max_pixel_count
  if (wants.target_pixel_count &&
      *wants.target_pixel_count >= wants.max_pixel_count) {
    wants.target_pixel_count.emplace(wants.max_pixel_count);
  }
  // 保存最终 wants
  current_wants_ = wants;
}

最终， VideoAdapter 更新 wants。

void VideoAdapter::OnSinkWants(const rtc::VideoSinkWants& sink_wants) {
  webrtc::MutexLock lock(&mutex_);
  // 更新 wants 请求的 最大分辨率像素数
  resolution_request_max_pixel_count_ = sink_wants.max_pixel_count;
  // 更新 wants 请求的目标分辨率像素数
  resolution_request_target_pixel_count_ =
      sink_wants.target_pixel_count.value_or(
          resolution_request_max_pixel_count_);
  // 更新 wants 请求的最大帧率
  max_framerate_request_ = sink_wants.max_framerate_fps;
  // 更新分辨率对齐字节数
  resolution_alignment_ = cricket::LeastCommonMultiple(
      source_resolution_alignment_, sink_wants.resolution_alignment);
}

• 使用设置的分辨率，处理采集帧

视频源（摄像头或屏幕共享）采集到视频帧，通过 ObjCVideoTrackSource::OnCapturedFrame()进行处理，并分发给各个Sinks。

void ObjCVideoTrackSource::OnCapturedFrame(RTC_OBJC_TYPE(RTCVideoFrame) * frame) {
  const int64_t timestamp_us = frame.timeStampNs / rtc::kNumNanosecsPerMicrosec;
  const int64_t translated_timestamp_us =
      timestamp_aligner_.TranslateTimestamp(timestamp_us, rtc::TimeMicros());

  int adapted_width;
  int adapted_height;
  int crop_width;
  int crop_height;
  int crop_x;
  int crop_y;
  // 调用基类 AdaptedVideoTrackSource 的实现获取视频帧裁剪数据，此时将用到上述流程设置的目标和最大分辨率像素数等值。
  if (!AdaptFrame(frame.width,
                  frame.height,
                  timestamp_us,
                  &adapted_width,
                  &adapted_height,
                  &crop_width,
                  &crop_height,
                  &crop_x,
                  &crop_y)) {
    return;
  }

  rtc::scoped_refptr<VideoFrameBuffer> buffer;
  if (adapted_width == frame.width && adapted_height == frame.height) {
    // No adaption - optimized path.
    // 不需要裁剪
    buffer = new rtc::RefCountedObject<ObjCFrameBuffer>(frame.buffer);
  } else if ([frame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]) {
    // Adapted CVPixelBuffer frame.
    // 需要裁剪，但是 RTCCVPixelBuffer 格式保存数据，将在 RTCCVPixelBuffer 内部根据裁剪参数进行处理
    RTC_OBJC_TYPE(RTCCVPixelBuffer) *rtcPixelBuffer =
        (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)frame.buffer;
    buffer = new rtc::RefCountedObject<ObjCFrameBuffer>([[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc]
        initWithPixelBuffer:rtcPixelBuffer.pixelBuffer
               adaptedWidth:adapted_width
              adaptedHeight:adapted_height
                  cropWidth:crop_width
                 cropHeight:crop_height
                      cropX:crop_x + rtcPixelBuffer.cropX
                      cropY:crop_y + rtcPixelBuffer.cropY]);
  } else {
    // Adapted I420 frame.
    // TODO(magjed): Optimize this I420 path.
    // 如果是 I420 帧格式，则直接进行裁剪
    rtc::scoped_refptr<I420Buffer> i420_buffer = I420Buffer::Create(adapted_width, adapted_height);
    buffer = new rtc::RefCountedObject<ObjCFrameBuffer>(frame.buffer);
    i420_buffer->CropAndScaleFrom(*buffer->ToI420(), crop_x, crop_y, crop_width, crop_height);
    buffer = i420_buffer;
  }

  // Applying rotation is only supported for legacy reasons and performance is
  // not critical here.
  VideoRotation rotation = static_cast<VideoRotation>(frame.rotation);
  if (apply_rotation() && rotation != kVideoRotation_0) {
    // 进行角度转换处理
    buffer = I420Buffer::Rotate(*buffer->ToI420(), rotation);
    rotation = kVideoRotation_0;
  }
  // 分发处理完之后的视频帧到各 sink
  OnFrame(VideoFrame::Builder()
              .set_video_frame_buffer(buffer)
              .set_rotation(rotation)
              .set_timestamp_us(translated_timestamp_us)
              .build());
}

获取视频帧裁剪参数。

bool AdaptedVideoTrackSource::AdaptFrame(int width,
                                         int height,
                                         int64_t time_us,
                                         int* out_width,
                                         int* out_height,
                                         int* crop_width,
                                         int* crop_height,
                                         int* crop_x,
                                         int* crop_y) {
  {
    webrtc::MutexLock lock(&stats_mutex_);
    stats_ = Stats{width, height};
  }

  if (!broadcaster_.frame_wanted()) {
    return false;
  }

  if (!video_adapter_.AdaptFrameResolution(
          width, height, time_us * rtc::kNumNanosecsPerMicrosec, crop_width,
          crop_height, out_width, out_height)) {
    broadcaster_.OnDiscardedFrame();
    // VideoAdapter dropped the frame.
    return false;
  }

  *crop_x = (width - *crop_width) / 2;
  *crop_y = (height - *crop_height) / 2;
  return true;
}

bool VideoAdapter::AdaptFrameResolution(int in_width,
                                        int in_height,
                                        int64_t in_timestamp_ns,
                                        int* cropped_width,
                                        int* cropped_height,
                                        int* out_width,
                                        int* out_height) {
  webrtc::MutexLock lock(&mutex_);
  ++frames_in_;

  // The max output pixel count is the minimum of the requests from
  // OnOutputFormatRequest and OnResolutionFramerateRequest.
  int max_pixel_count = resolution_request_max_pixel_count_;

  // Select target aspect ratio and max pixel count depending on input frame
  // orientation.
  // 获取 resolution_request_max_pixel_count_ 与 max_portrait_pixel_count_中的最小值，
  // 由此可见，通过 OnOutputFormatRequest() 设置进来的 max_portrait_pixel_count_ 限制了最大编码分辨率
  absl::optional<std::pair<int, int>> target_aspect_ratio;
  if (in_width > in_height) {
    target_aspect_ratio = target_landscape_aspect_ratio_;
    if (max_landscape_pixel_count_)
      max_pixel_count = std::min(max_pixel_count, *max_landscape_pixel_count_);
  } else {
    target_aspect_ratio = target_