WebRTC系列-移动端硬件编码支持Simulcast

写在开头: 本篇文章扩展自Simulcast这篇文章，这篇文章只写了个大概，这里细说实现细节;以下的功能实现都是在M76版本源码。

联播在H264EncoderImpl 实现
在移动端的WebRTC的native源码中H264有两种实现：基于系统的硬件支持的h264和在h264_encoder_impl.cc(video_coding/codecs/h264/)中实现的openH264软件编码;
硬件的Simulcast功能除了本文的描述有一种(这种方案已经在实际羡慕中测试验证),软件H264的实现是基于FFmpeg和openH264的实现：代码可以参考 ;

类和编码器的初始化
 

int32_t H264EncoderImpl::InitEncode(const VideoCodec* inst,
                                    int32_t number_of_cores,
                                    size_t max_payload_size) {
 // ReportInit();
 // 省略一些参数检查代码

// 下面两行代码 获取联播的个数
  int number_of_streams = SimulcastUtility::NumberOfSimulcastStreams(*inst);
  bool doing_simulcast = (number_of_streams > 1);
// 验证联播参数是否合法
  if (doing_simulcast &&
      !SimulcastUtility::ValidSimulcastParameters(*inst, number_of_streams)) {
    return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED;
  }
 // 保存一些数据到变量 保存在configurations_
  num_temporal_layers_ = codec_.H264()->numberOfTemporalLayers;

  for (int i = 0, idx = number_of_streams - 1; i < number_of_streams;
       ++i, --idx) {
    ISVCEncoder* openh264_encoder;
    // 创建编码器
    if (WelsCreateSVCEncoder(&openh264_encoder) != 0) {
      // Failed to create encoder.
     // 省略代码
      return WEBRTC_VIDEO_CODEC_ERROR;
    }
    // 将编码器存储在数组中
    encoders_[i] = openh264_encoder;
    // 当simucast层数大于1的时候创建缩小的图像需要缓冲区.
    // 设置开始码率(startBitrate)和最大码率(maxBitrate) 保存在configurations_
   
    //根据每个联播层的配置创建编码器参数。从configurations_中取
    SEncParamExt encoder_params = CreateEncoderParams(i);
    // 初始化编码器
    if (openh264_encoder->InitializeExt(&encoder_params) != 0) {
     // 省略代码
      return WEBRTC_VIDEO_CODEC_ERROR;
    }
    // 省略代码
    ...
  return WEBRTC_VIDEO_CODEC_OK;
}

从上述源码中的 H264EncoderImpl::InitEncode函数可以看到, 真正负责编码工作的对象是 ISVCEncoder* openh264_encoder，在InitEncode函数中如果有多个联播层就去循环创建编码器及初始化编码器，准备联播层缩放缓冲区等参数的初始化;
编码是在Encode函数中：

Simucast编码
编码实现比较复杂代码量较大这里只看核心处理代码:
 

int32_t H264EncoderImpl::Encode(
    const VideoFrame& input_frame,
    const std::vector<VideoFrameType>* frame_types) {
    //代码省略
    ...
  // 遍历存储的编码器，编码对应每一个视频层
  for (size_t i = 0; i < encoders_.size(); ++i) {
    //准备编码输入帧
    pictures_[i] = {0};
   //代码省略
    ...
    // Downscale images on second and ongoing layers.
    if (i == 0) { //代码省略
      ...
    }else{ //缩小第二层和正在进行的层上的图像。
      //代码省略
      ...
    }
    //如果配置次层不需要发送则忽略
    if (!configurations_[i].sending) {continue; }
    //编码关键帧
    if (send_key_frame) {
      //代码省略
      ...=
	    int enc_ret = encoders_[i]->EncodeFrame(&pictures_[i], &info);
	    if (enc_ret != 0) { return WEBRTC_VIDEO_CODEC_ERROR;
    }
    // 将编码图像分割成片段,同时也更新
    // |encoded_image_|.
    RTPFragmentationHeader frag_header;
    RtpFragmentize(&encoded_images_[i], *frame_buffer, &info, &frag_header);

    // 在这种情况下，编码器可以跳过帧以节省带宽
    // |encoded_images_[i]._length| == 0.
    if (encoded_images_[i].size() > 0) {
      //代码省略
      ...=
      encoded_image_callback_->OnEncodedImage(encoded_images_[i],
                                              &codec_specific, &frag_header);
    }
  }
  return WEBRTC_VIDEO_CODEC_OK;
}

上述就是编码多个编码器编码多路视频的简述代码，具体代码可以参看源码实现;

在 Android/iOS 等移动端, H264EncoderImpl 实现的 Simulcast 因 OpenH264 软件编码的缘故, 往往有CPU算力和电池续航方面的不足.

另外, WebRTC iOS编译 FFmpeg 和 OpenH264 依然是一个巨坑. 那么有没有利用硬件编码实现的Simulcast呢.

上述是原文中的话，iOS端在尝试把 rtc_use_h264设置成true，但是会出现其他问题,在谷歌谈论组 WebRTC IOS h264 support!，中有描述和实际编译遇到的问题一致；大致如图:

在webrtc中关于联播还有一个适配器的实现SimulcastEncoderAdapter

 

SimulcastEncoderAdapter 实现

 

基于适配器类实现硬件联播

在chromium blink的代码中有一个类似的实现，具体可以参见Chromium 源码

创建相应工厂

参照blink实现创建工厂类,.h文件：

#include <memory>


#include "rtc_base/system/rtc_export.h"
#include "api/video_codecs/video_encoder_factory.h"


namespace webrtc {

// Creates a new factory that can create the built-in types of video encoders.
// The factory has simulcast support for VP8.
RTC_EXPORT std::unique_ptr<VideoEncoderFactory>
CreateVideoEncoderAdapterFactory(std::unique_ptr<VideoEncoderFactory> hardware_encoder_factory);

}  // namespace webrtc

.m文件实现:

#include "video/video_codec_factory_adapter.h"
//#include "base/memory/scoped_refptr.h"
//#include "base/memory/ptr_util.h"

#include "build/build_config.h"
//#include "media/base/media_switches.h"
//#include "media/engine/internal_decoder_factory.h"
#include "api/video_codecs/video_encoder_software_fallback_wrapper.h"
#include "media/engine/simulcast_encoder_adapter.h"
#include <vector>

#include "absl/memory/memory.h"
#include "absl/strings/match.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_encoder.h"
#include "media/base/codec.h"
#include "media/base/media_constants.h"
#include "media/engine/encoder_simulcast_proxy.h"
#include "media/engine/internal_encoder_factory.h"
#include "rtc_base/checks.h"


namespace webrtc {
//template<typename T, typename... Ts>
//std::unique_ptr<T> make_unique(Ts&&... params)
//{
//    return std::unique_ptr<T>(new T(std::forward<Ts>(params)...));
//}
namespace {

template <typename Factory>
bool IsFormatSupported(const Factory* factory,
                       const webrtc::SdpVideoFormat& format) {
  return factory && format.IsCodecInList(factory->GetSupportedFormats());
}

// Merge |formats1| and |formats2|, but avoid adding duplicate formats.
std::vector<webrtc::SdpVideoFormat> MergeFormats(
    std::vector<webrtc::SdpVideoFormat> formats1,
    const std::vector<webrtc::SdpVideoFormat>& formats2) {
  for (const webrtc::SdpVideoFormat& format : formats2) {
    // Don't add same format twice.
    if (!format.IsCodecInList(formats1))
      formats1.push_back(format);
  }
  return formats1;
}
//bool IsFormatSupported(
//                       const Factory* factory,
//                       const webrtc::SdpVideoFormat& format) {
//  return factory && IsFormatSupported(factory->GetSupportedFormats(), format);
//}
//
 Merge |formats1| and |formats2|, but avoid adding duplicate formats.
//std::vector<webrtc::SdpVideoFormat> MergeFormats(
//    std::vector<webrtc::SdpVideoFormat> formats1,
//    const std::vector<webrtc::SdpVideoFormat>& formats2) {
//  for (const webrtc::SdpVideoFormat& format : formats2) {
//    // Don't add same format twice.
//    if (!IsFormatSupported(formats1, format))
//      formats1.push_back(format);
//  }
//  return formats1;
//}

// This class combines a hardware factory with the internal factory and adds
// internal SW codecs, simulcast, and SW fallback wrappers.
class EncoderAdapter : public webrtc::VideoEncoderFactory {
 public:
   EncoderAdapter(
      std::unique_ptr<webrtc::VideoEncoderFactory> hardware_encoder_factory)
      : hardware_encoder_factory_(std::move(hardware_encoder_factory)) {}

  webrtc::VideoEncoderFactory::CodecInfo QueryVideoEncoder(
      const webrtc::SdpVideoFormat& format) const override {
    const webrtc::VideoEncoderFactory* factory =
          IsFormatSupported(hardware_encoder_factory_.get(), format)
            ? hardware_encoder_factory_.get()
            : &software_encoder_factory_;
    return factory->QueryVideoEncoder(format);
  }

    std::unique_ptr<webrtc::VideoEncoder> CreateVideoEncoder(
      const webrtc::SdpVideoFormat& format) override {
    const bool supported_in_software =
        IsFormatSupported(&software_encoder_factory_, format);
    const bool supported_in_hardware =
        IsFormatSupported(hardware_encoder_factory_.get(), format);

    if (!supported_in_software && !supported_in_hardware)
      return nullptr;

    if (absl::EqualsIgnoreCase(format.name.c_str(),
                                         cricket::kVp9CodecName)) {
      // For VP9 and AV1 we don't use simulcast.
      // return software_encoder_factory_.CreateVideoEncoder(format);
      return hardware_encoder_factory_->CreateVideoEncoder(format);
    }

    if (!supported_in_hardware || !hardware_encoder_factory_.get()) {
      return absl::make_unique<webrtc::SimulcastEncoderAdapter>(
          &software_encoder_factory_, format);
    } else if (!supported_in_software) {
    //这里原来使用std::make_unique，这个api在c++14 中，所以这里替换成absl中的 实现
      return absl::make_unique<webrtc::SimulcastEncoderAdapter>(
          hardware_encoder_factory_.get(), format);
    }

    return absl::make_unique<webrtc::SimulcastEncoderAdapter>(&software_encoder_factory_, format);
  }

  std::vector<webrtc::SdpVideoFormat> GetSupportedFormats() const override {
    std::vector<webrtc::SdpVideoFormat> software_formats =
        software_encoder_factory_.GetSupportedFormats();
    return hardware_encoder_factory_
               ? MergeFormats(software_formats,
                              hardware_encoder_factory_->GetSupportedFormats())
               : software_formats;
  }

 private:
  webrtc::InternalEncoderFactory software_encoder_factory_;
  const std::unique_ptr<webrtc::VideoEncoderFactory> hardware_encoder_factory_;
};
} // namespace
std::unique_ptr<VideoEncoderFactory> CreateVideoEncoderAdapterFactory(
  std::unique_ptr<webrtc::VideoEncoderFactory> hardware_encoder_factory) {
  return absl::make_unique<EncoderAdapter>(std::move(hardware_encoder_factory));
}

}  // namespace webrtc

注意：SimulcastEncoderAdapter在最新代码中是有三个参数的构造方法;

使用方法

//  ios 使用
        auto video_encoder_factory = std::move(native_encoder_factory);
        return [self initWithNativeAudioEncoderFactory:webrtc::CreateBuiltinAudioEncoderFactory()
                             nativeAudioDecoderFactory:webrtc::CreateBuiltinAudioDecoderFactory()
                             nativeVideoEncoderFactory:webrtc::CreateVideoEncoderAdapterFactory(std::move(video_encoder_factory))
                             nativeVideoDecoderFactory:webrtc::CreateBuiltinVideoDecoderFactory()
                                     audioDeviceModule:[self audioDeviceModule]
                                 audioProcessingModule:nullptr];
// android 使用
  cricket::MediaEngineDependencies media_dependencies;
  media_dependencies.task_queue_factory = dependencies.task_queue_factory.get();
  media_dependencies.adm = std::move(audio_device_module);
  media_dependencies.audio_encoder_factory = std::move(audio_encoder_factory);
  media_dependencies.audio_decoder_factory = std::move(audio_decoder_factory);
  media_dependencies.audio_processing = std::move(audio_processor);
  auto video_encoder_factory =
      absl::WrapUnique(CreateVideoEncoderFactory(jni, jencoder_factory));
  media_dependencies.video_encoder_factory = 
        CreateVideoEncoderAdapterFactory(std::move(video_encoder_factory));
  media_dependencies.video_decoder_factory =
      absl::WrapUnique(CreateVideoDecoderFactory(jni, jdecoder_factory));
  dependencies.media_engine =
      cricket::CreateMediaEngine(std::move(media_dependencies));

  rtc::scoped_refptr<PeerConnectionFactoryInterface> factory =
      CreateModularPeerConnectionFactory(std::move(dependencies));