1、前言
本文是webrtc中拥塞控制的上文,主要是分析webrtc中的拥塞控制的码率探测,预估和调整的部分,介绍了整体框架和原理以及相关的类;webrtc版本:M91
2、正文
2.1 整体框架
webrtc中的部分码控结构如下图所示,从socket层接收到数据后,到transport解析rtcp包处理得到feedback,通过call将feedback转发到对应sendstream上的rtcp处理模块,最终通过RtpTransportControllerSend将feedback转发到GoogCcNetworkController进行码率预估后,把预估的码率(target bitrate), 探测策略(probe config), congestion windows给pacer,pacer转发给pacingContrller去使用进行发送码率控制
其中以GoogCcNetworkController作为整个码率预估及调整的核心 ,涉及的类和流程如下图所示,红框中的类在GoogCcNetworkController下
接下来会以GoogCcNetworkController的码率预估过程为例, 详细介绍webrtc中带宽控制的架构和过程。
2.2 GoogCcNetworkController
GoogCcNetworkController是码率预估的核心类, 如2.1中所示的webrtc中的部分码控结构上,可以看到其所属于class RtpTransportControllerSend
2.2.1 GoogCcNetworkController创建时刻
在底层网络可用的时候,会触发RtpTransportControllerSend::OnNetworkAvailability()回调
void RtpTransportControllerSend::OnNetworkAvailability(bool network_available) {
RTC_LOG(LS_VERBOSE) << "SignalNetworkState "
<< (network_available ? "Up" : "Down");
NetworkAvailability msg;
msg.at_time = Timestamp::Millis(clock_->TimeInMilliseconds());
msg.network_available = network_available;
task_queue_.PostTask([this, msg]() {
RTC_DCHECK_RUN_ON(&task_queue_);
if (network_available_ == msg.network_available)
return;
network_available_ = msg.network_available;
if (network_available_) {
pacer()->Resume();
} else {
pacer()->Pause();
}
pacer()->UpdateOutstandingData(DataSize::Zero());
if (controller_) {
control_handler_->SetNetworkAvailability(network_available_);
PostUpdates(controller_->OnNetworkAvailability(msg));
UpdateControlState();
} else {
// 未创建controller,创建
MaybeCreateControllers();
}
});
for (auto& rtp_sender : video_rtp_senders_) {
rtp_sender->OnNetworkAvailability(network_available);
}
}
其检测到未创建controller_时,会调用 RtpTransportControllerSend::MaybeCreateControllers()创建
void RtpTransportControllerSend::MaybeCreateControllers() {
RTC_DCHECK(!controller_);
RTC_DCHECK(!control_handler_);
if (!network_available_ || !observer_)
return;
control_handler_ = std::make_unique<congestioncontrolhandler>();
initial_config_.constraints.at_time =
Timestamp::Millis(clock_->TimeInMilliseconds());
initial_config_.stream_based_config = streams_config_;
// TODO(srte): Use fallback controller if no feedback is available.
// 创建GoogCcNetworkController
if (controller_factory_override_) {
RTC_LOG(LS_INFO) << "Creating overridden congestion controller";
controller_ = controller_factory_override_->Create(initial_config_);
process_interval_ = controller_factory_override_->GetProcessInterval();
} else {
RTC_LOG(LS_INFO) << "Creating fallback congestion controller";
controller_ = controller_factory_fallback_->Create(initial_config_);
process_interval_ = controller_factory_fallback_->GetProcessInterval();
}
// 间隔更新GoogCcNetworkController
UpdateControllerWithTimeInterval();
StartProcessPeriodicTasks();
}
创建后即刻就调用 UpdateControllerWithTimeInterval() 和 StartProcessPeriodicTasks():
void RtpTransportControllerSend::UpdateControllerWithTimeInterval() {
RTC_DCHECK(controller_);
ProcessInterval msg;
msg.at_time = Timestamp::Millis(clock_->TimeInMilliseconds());
if (add_pacing_to_cwin_)
msg.pacer_queue = pacer()->QueueSizeData();
// 对码率进行检测和更新,将结果转发给pacer
PostUpdates(controller_->OnProcessInterval(msg));
}
UpdateControllerWithTimeInterval()中:
调用GoogCcNetworkController::OnProcessInterval()做间隔的码率检测和更新
调用PostUpdates()将最新的码率给转发到pacer
void RtpTransportControllerSend::StartProcessPeriodicTasks() {
if (!pacer_queue_update_task_.Running()) {
pacer_queue_update_task_ = RepeatingTaskHandle::DelayedStart(
task_queue_.Get(), kPacerQueueUpdateInterval, [this]() {
RTC_DCHECK_RUN_ON(&task_queue_);
TimeDelta expected_queue_time = pacer()->ExpectedQueueTime();
control_handler_->SetPacerQueue(expected_queue_time);
UpdateControlState();
return kPacerQueueUpdateInterval;
});
}
controller_task_.Stop();
if (process_interval_.IsFinite()) {
// 定时检测更新码率
controller_task_ = RepeatingTaskHandle::DelayedStart(
task_queue_.Get(), process_interval_, [this]() {
RTC_DCHECK_RUN_ON(&task_queue_);
UpdateControllerWithTimeInterval();
return process_interval_;
});
}
}
StartProcessPeriodicTasks()中:
对control_handler_进行了更新,control_handler 是一个将controller计算相关码率信息路由回调给其它模块的一个类(后续在仔细分析),调用UpdateControlState()更新,将信息回调给其它
创建了一个controller_task_去定时的做UpdateControllerWithTimeInterval() 接下来会通过介绍cc-controller下最重要的几个函数来介绍码率控制的核心过程,其分别是OnProcessInterval()和OnTransportPacketsFeedback(),前者根据时间流逝定时更新码率, 后者需要借助于cc-feedback的到来才能更新码率, 这两个函数涉及到的类都很广,如果把里面的类一次性介绍到底的话,文章的逻辑结构性会很差,所以把其中涉及到的类都提出来点到为止,详细的会放在后面去独立介绍,可自行查阅。
2.2.2 定时检测-OnProcessInterval()
GoogCcNetworkController::OnProcessInterval()是cc-controller的核心函数之一,会定时的触发,用来做带宽检测和更新:
NetworkControlUpdate GoogCcNetworkController::OnProcessInterval(
ProcessInterval msg) {
NetworkControlUpdate update;
if (initial_config_) {
// 重设loss_based和delay_based码率探测器和probe的初始码率
// 获得码率探测簇配置(probe_cluster_config)
update.probe_cluster_configs =
ResetConstraints(initial_config_->constraints);
// 获取当前pacing 的发送码率, padding, time_windows等
update.pacer_config = GetPacingRates(msg.at_time);
// probe探测完成后,允许其因为alr需要快速恢复码率而继续做probe
if (initial_config_->stream_based_config.requests_alr_probing) {
probe_controller_->EnablePeriodicAlrProbing(
*initial_config_->stream_based_config.requests_alr_probing);
}
absl::optional<datarate> total_bitrate =
initial_config_->stream_based_config.max_total_allocated_bitrate;
if (total_bitrate) {
// 为probe设置最大的分配码率(MaxTotalAllocatedBitrate)作为探测的上边界
// 并生成响应的probe_cluster_config去进行探测
auto probes = probe_controller_->OnMaxTotalAllocatedBitrate(
total_bitrate->bps(), msg.at_time.ms());
update.probe_cluster_configs.insert(update.probe_cluster_configs.end(),
probes.begin(), probes.end());
max_total_allocated_bitrate_ = *total_bitrate;
}
// 释放initial_config_,下次进来就不通过init_config做初始化了
initial_config_.reset();
}
// 更新拥塞窗口中的pacing数据长度
if (congestion_window_pushback_controller_ && msg.pacer_queue) {
congestion_window_pushback_controller_->UpdatePacingQueue(
msg.pacer_queue->bytes());
}
// 更新码率
bandwidth_estimation_->UpdateEstimate(msg.at_time);
// 检测当前是否处于alr
absl::optional<int64_t> start_time_ms =
alr_detector_->GetApplicationLimitedRegionStartTime();
// 如果处于alr,告诉probe_controller处于alr,可以进行探测,进行快恢复
probe_controller_->SetAlrStartTimeMs(start_time_ms);
// 检测当前是否因alr状态而需要做probe了,获取probe_cluster_config
auto probes = probe_controller_->Process(msg.at_time.ms());
update.probe_cluster_configs.insert(update.probe_cluster_configs.end(),
probes.begin(), probes.end());
if (rate_control_settings_.UseCongestionWindow() &&
last_packet_received_time_.IsFinite() && !feedback_max_rtts_.empty()) {
// 根据rtt和target_rate 更新当前拥塞控制窗口大小
UpdateCongestionWindowSize();
}
if (congestion_window_pushback_controller_ && current_data_window_) {
// 重新设置拥塞控制窗口大小
congestion_window_pushback_controller_->SetDataWindow(
*current_data_window_);
} else {
update.congestion_window = current_data_window_;
}
// 获取更新后的码率,probe等,同时对alr, probe_controller中的码率进行更新
MaybeTriggerOnNetworkChanged(&update, msg.at_time);
return update;
}
GoogCcNetworkController::OnProcessInterval()中:
在第一次调用该函数时,使用initial_config_设置DelayBasedBwe, SendSideBandwidthEstimation, ProbeController中的初始码率,ProbeController设置完码率之后会返回一个probe_cluster_config(探测簇), probe_cluster_config会返回给pacing_controller,pacing_controller在发包的时候使用其中的码率去发包以配合码率探测。
为ProbeController设置最大分配码率(MaxTotalAllocatedBitrate),这个值在ProbeController中会被用来做探测的上边界,一旦探测的码率到达这个值,就停止普通探测。
过了初始化后,SendSideBandwidthEstimation(也就是bandwidth_estimation_)会基于时间更新码率,其内部虽然是依靠cc-feedback提供丢包率来预估码率,当没有feedback也会基于时间预估当前的rtt去更新码率。
从AlrDetector获取当前是否处于alr状态,AlrDetector在每次发送数据时(OnSentPacket)都会检测实际发送码率是否与目标码率相差太多悬殊,从而判断是否(受限于编码器等原因而导致)无法达到目标码率,从而设定处于alr状态,alr状态非常有用,带宽预测的核心是需要向链路中发送足够的包去观察链路情况,如果探测到处于alr状态无法达到这个要求,就需要一些额外手段去处理。
设置ProbeController处于alr状态。ProbeController内完整了初始的在正常探测后就不再探测了,但如果处于alr状态或者网络变化的状态,是需要对网络进行探测以便于网络的快恢复;
从ProbeController获取probe_cluster_config,以进行需要可能的探测
根据rtt和congestion重新计算拥塞窗口控制器中的的数据大小(CongestionWindowPushbackController)
bandwidth_estimation_可能对码率进行了更新,调用MaybeTriggerOnNetworkChanged()将更新的码率同步到alr,probe_controller中,同时将码率,probe_config等放到update中返回。
2.2.3 cc-feedback
2.2.3.1 cc-feedback报文
在介绍cc-controler中另一个重要的函数OnTransportPacketsFeedback()前,因其在收到cc-feedback时触发。所以先介绍cc-feedback,cc-feedback协议的设计和详情可见R2. transport-cc-feedback草案或R5. WebRTC研究:Transport-cc之RTP及RTCP, 都介绍的非常详细易懂。
简单从报文介绍一下我们能从cc-feedback拿到什么:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|V=2|P| FMT=15 | PT=205 | length |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
0 | SSRC of packet sender |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
4 | SSRC of media source |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
8 | base sequence number | packet status count |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
12 | reference time | fb pkt. count |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
16 | packet chunk | packet chunk |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
. .
. .
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| packet chunk | recv delta | recv delta |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
. .
. .
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| recv delta | recv delta | zero padding |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
cc-feedback的PT=205, FMT=15, 从base sequence number开始就是cc-feedback的报文主体:
base sequence number:TransportFeedback包中记录的第一个RTP包的transport sequence number
packet status count: 表示这个TransportFeedback包记录了多少个RTP包信息
reference time: 基准时间,以64ms为单位,可以和下面的recv delta求和得到包的接收时间
fb pkt. count: 当前feedback的序列号,用于检测cc-feedback是否丢包
后面会跟着两个数组,代表着transport number以base sequence number为基准递增的包的相关信息
packet chunk: 当前包的到达状态(到达\丢失),
recv delta: 接收时间delta,要和reference time求和才能得到真正的接收时间。
可以看到cc-feedback中能得到包的接收状态和时间。
2.2.3.2 transprot-sequence-number
对于cc-feedback,说明一下webrtc的整体处理过程。
webrtc为每个rtp packet添加了一个transport-cc number的rtp extension用来标识每个包的传输序列号,见官方草案描述:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| 0xBE | 0xDE | length=1 |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| ID | L=1 |transport-wide sequence number | zero padding |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
添加该number的主要是分离媒体(使用sequence number)和网络处理(使用transport number)。
在RTPSenderVideo::SendVideo()中使用AllocatePacket()为每帧的数据生成rtp packet的时,默认会为当前packet保留一些rtp-extension, 其中就包括了TransportSequenceNumber。
std::unique_ptr<rtppackettosend> RTPSender::AllocatePacket() const {
...
// Reserve extensions, if registered, RtpSender set in SendToNetwork.
packet->ReserveExtension<absolutesendtime>();
packet->ReserveExtension<transmissionoffset>();
packet->ReserveExtension<transportsequencenumber>();//<----
...
}
extension register
(接下来这段介绍的是extension 的register过程,不感兴趣的可以不看)
正如上面的AllocatePacket()中的注释所言,保存这些extension,如果这些extension注册了,那么RtpSender中会对这些extension进行设值; extension Register的过程要从RtpVideoSender溯源,其初始化时将将传入的rtp_config.extension设置到了每个stream的rtp_rtcp中
RtpVideoSender::RtpVideoSender(....){ // 实在太长了,省略一些参数,而不是一个变参构造函数
...
// RTP/RTCP initialization.
for (size_t i = 0; i < rtp_config_.extensions.size(); ++i) {
const std::string& extension = rtp_config_.extensions[i].uri;
// 将rtp_config中的所有extension设置到stream对应的rtp_rtcp module下
int id = rtp_config_.extensions[i].id;
RTC_DCHECK(RtpExtension::IsSupportedForVideo(extension));
for (const RtpStreamSender& stream : rtp_streams_) {
// rtp_rtcp module注册这些extension
stream.rtp_rtcp->RegisterRtpHeaderExtension(extension, id);
}
}
...
}
rtp_rtcp将其转发到packet_generator(实则RTPSender)
void ModuleRtpRtcpImpl2::RegisterRtpHeaderExtension(absl::string_view uri,
int id) {
// 转发到packet_generator
bool registered =
rtp_sender_->packet_generator.RegisterRtpHeaderExtension(uri, id);
RTC_CHECK(registered);
}
RTPSender注册该extension, 然后会看到一个很重要的变量supports_bwe_extension_会被HasBweExtension()检测更新,根据是否已注册了所有的bwe extension设置为true,这个变量决定能否使用padding功能(带宽探测时,当前数据量达不到目标发送码率,用一些历史包或者空数据做带宽填充)
bool RTPSender::RegisterRtpHeaderExtension(absl::string_view uri, int id) {
MutexLock lock(&send_mutex_);
bool registered = rtp_header_extension_map_.RegisterByUri(id, uri);// 注册该extension
supports_bwe_extension_ = HasBweExtension(rtp_header_extension_map_);
UpdateHeaderSizes();
return registered;
}
// bwe所需extension
bool HasBweExtension(const RtpHeaderExtensionMap& extensions_map) {
return extensions_map.IsRegistered(kRtpExtensionTransportSequenceNumber) ||
extensions_map.IsRegistered(kRtpExtensionTransportSequenceNumber02) ||
extensions_map.IsRegistered(kRtpExtensionAbsoluteSendTime) ||
extensions_map.IsRegistered(kRtpExtensionTransmissionTimeOffset);
}
//padding的支持需要bwe extension
bool RTPSender::SupportsPadding() const {
MutexLock lock(&send_mutex_);
return sending_media_ && supports_bwe_extension_;
}
extension register介绍到此为止,看完了整个过程也没有很明确的找到按照注释所言--"发现注册了这个extension,然后才对它这个extension设值"的处理, 但是还是提及到了一些重要的probing的东西
2.2.3.3 feedback packet的生成
最后在paced发送packet的过程中,当packet到达PacketRouter时,将会检测其是否有TransportSequenceNumber, 如果有则将transport_sequence_number设置到到packet的头部
void PacketRouter::SendPacket(std::unique_ptr<rtppackettosend> packet,
const PacedPacketInfo& cluster_info) {
...
MutexLock lock(&modules_mutex_);
// 设置transpoort sequence number
if (packet->HasExtension<transportsequencenumber>()) {
packet->SetExtension<transportsequencenumber>((++transport_seq_) & 0xFFFF);
}
...
}
之后,当packet经过RtpSenderEgress模块的时,在RtpSenderEgress::SendPacket()中会提取其transport_sequence_number生成feedback包,整个流程如下:
void RtpSenderEgress::SendPacket(RtpPacketToSend* packet,
const PacedPacketInfo& pacing_info) {
...
if (auto packet_id = packet->GetExtension<transportsequencenumber>()) {
options.packet_id = *packet_id;
options.included_in_feedback = true;
options.included_in_allocation = true;
// 添加该packet到feedback
AddPacketToTransportFeedback(*packet_id, *packet, pacing_info);
}
...
}
构造packet_info,通知feedback_ovserver添加该包
voidRtpSenderEgress::AddPacketToTransportFeedback(
uint16_tpacket_id,
constRtpPacketToSend&packet,
constPacedPacketInfo&pacing_info) {
if (transport_feedback_observer_) {
size_tpacket_size=packet.payload_size() +packet.padding_size();
if (send_side_bwe_with_overhead_) {
packet_size=packet.size();
}
// 构造packet_info
RtpPacketSendInfopacket_info;
packet_info.ssrc=ssrc_;
packet_info.transport_sequence_number=packet_id;
packet_info.rtp_sequence_number=packet.SequenceNumber();
packet_info.length=packet_size;
packet_info.pacing_info=pacing_info;
packet_info.packet_type=packet.packet_type();
// 通知feedback_ovserver添加该包
transport_feedback_observer_->OnAddPacket(packet_info);
}
}
告知RtpTransportControllerSend有包发送了, 调用transport_feedbadck_adapter_为其生成feedback包
void RtpTransportControllerSend::OnAddPacket(
const RtpPacketSendInfo& packet_info) {
feedback_demuxer_.AddPacket(packet_info);
Timestamp creation_time = Timestamp::Millis(clock_->TimeInMilliseconds());
task_queue_.PostTask([this, packet_info, creation_time]() {
RTC_DCHECK_RUN_ON(&task_queue_);
// 往adapter_添加feedback
transport_feedback_adapter_.AddPacket(
packet_info,
send_side_bwe_with_overhead_ ? transport_overhead_bytes_per_packet_ : 0,
creation_time);
});
}
TransportFeedbackAdapter生成feedback packet,将其存入history_中
void TransportFeedbackAdapter::AddPacket(const RtpPacketSendInfo& packet_info,
size_t overhead_bytes,
Timestamp creation_time) {
// 生成feedback包
PacketFeedback packet;
packet.creation_time = creation_time;
packet.sent.sequence_number =
seq_num_unwrapper_.Unwrap(packet_info.transport_sequence_number);
packet.sent.size = DataSize::Bytes(packet_info.length + overhead_bytes);
packet.sent.audio = packet_info.packet_type == RtpPacketMediaType::kAudio;
packet.network_route = network_route_;
packet.sent.pacing_info = packet_info.pacing_info;
while (!history_.empty() &&
creation_time - history_.begin()->second.creation_time >
kSendTimeHistoryWindow) {
// TODO(sprang): Warn if erasing (too many) old items?
if (history_.begin()->second.sent.sequence_number > last_ack_seq_num_)
in_flight_.RemoveInFlightPacketBytes(history_.begin()->second);
history_.erase(history_.begin());
}
// 以transport_sequence_number和packet为key-valiue,存入history_中
history_.insert(std::make_pair(packet.sent.sequence_number, packet));
}
2.2.3.4 feedback packet再赋值
在收到cc-feedback的rtcp包的时候,会经过层层转发到RTCPReceiver,
void RTCPReceiver::IncomingPacket(rtc::ArrayView<const uint8_t=""> packet) {
if (packet.empty()) {
RTC_LOG(LS_WARNING) << "Incoming empty RTCP packet";
return;
}
PacketInformation packet_information;
// 解析rtcp
if (!ParseCompoundPacket(packet, &packet_information))
return;
// 转发
TriggerCallbacksFromRtcpPacket(packet_information);
}
RTCPReceiver::IncomingPacket()中:
使用ParseCompoundPacket()对报文进行解析, ParseCompoundPacket()是一个非常精华的函数,可以再里面找到所有有关的RTCP包的解析(RR,SR,SDES, NACK, CC-FeedBack, Pli, Fir等),其内部会调用HandleTransportFeedback()将cc-feedback解析成transport_feedback,放到packet-information中
void RTCPReceiver::HandleTransportFeedback(
const CommonHeader& rtcp_block,
PacketInformation* packet_information) {
// 解析rtcp_block 生成transport_feedback
std::unique_ptr<rtcp::transportfeedback> transport_feedback(
new rtcp::TransportFeedback());
if (!transport_feedback->Parse(rtcp_block)) {
++num_skipped_packets_;
return;
}
packet_information->packet_type_flags |= kRtcpTransportFeedback;
packet_information->transport_feedback = std::move(transport_feedback);
}
然后调用TriggerCallbacksFromRtcpPacket()去转发该RTCP包.
TriggerCallbacksFromRtcpPacket()中会将解析出来的transport_feedback转发到RtpTransportControllerSend
void RTCPReceiver::TriggerCallbacksFromRtcpPacket(
const PacketInformation& packet_information) {
...
if (transport_feedback_observer_ &&
(packet_information.packet_type_flags & kRtcpTransportFeedback)) {
uint32_t media_source_ssrc =
packet_information.transport_feedback->media_ssrc();
if (media_source_ssrc