前言:
webrtc拥塞控制内容很多,目前只是主要研究了其基本机理,后续在仔细研究每个模块的机制,其控制拥塞的机理,主要还是平滑发送码率 ,调节发送码率,而码率调节的方式主要有三种:
1)基于丢包的码率控制
2) 基于接收端计算的码率反馈(通过remb方式反馈)
3) 基于发送端transport-wide-cc估计发送带宽的码率控制
后2种计算主要是基于延迟得码率控制,通过观测rtt得变化,计算延迟得变化情况从而估计当前网络拥塞得情况,2和3得主要区别在于一个在接收端计算,一个在发送端计算,其估算延迟变化所采用得滤波器也不同。
在开启发送估算特性,协商完sdp后,会在发送rtcp时增加相应扩展头,并且发送端会基于上述三种码率反馈,综合计算出一个合适的码率,通过相应的方式作用于不同的模块(pacer模块,编码模块?)
本篇文章的目的是先梳理该从哪些角度入手,需要弄清除哪些机制?后续几篇文章在代码精读。
整个拥塞控制涉及到的主要模块有
modules/bitrate_controller 码率控制
modules/congestion_controller 拥塞控制
modeules/remote_bitrate_estimator 远端码率估算
这是主要模块,牵连模块 有pacing call 等其它模块
一.GCC算法草案
https://c3lab.poliba.it/images/6/65/Gcc-analysis.pdf
https://zhuanlan.zhihu.com/p/87622467
https://www.cnblogs.com/wangyiyunxin/p/11122003.html
详述可以参考链接文章 ,这里不做过多解释,网上可参考的Blog也有很多。不过建议还是通读一遍英文或者相关rfc。
二.基于丢包的码率计算
先介绍下rtcp收包到计算丢包率估算码率的流程
关注rr报文中丢包率(fraction)和累计丢包总数。 fraction_lost是以256为单位值得出得数字。
从调用栈可以看出在收到rr报文时会去计算相关的丢包率,根据丢包率计算码率,下面主要分析下下面2个流程
//入参 blocks报文列表
//入参 rtt
//当前时间戳
void BitrateControllerImpl::OnReceivedRtcpReceiverReport(
const ReportBlockList& report_blocks,
int64_t rtt,
int64_t now_ms) {
if (report_blocks.empty()) //空返回
return;
{
rtc::CritScope cs(&critsect_);
int fraction_lost_aggregate = 0; //累计丢包数
int total_number_of_packets = 0; //包数
// Compute the a weighted average of the fraction loss from all report
// blocks.
//从报告块计算丢包率
for (const RTCPReportBlock& report_block : report_blocks) {
std::map<uint32_t, uint32_t>::iterator seq_num_it = //每路ssrc最后一次收到的最大序列号
ssrc_to_last_received_extended_high_seq_num_.find(
report_block.source_ssrc);
int number_of_packets = 0;
if (seq_num_it != ssrc_to_last_received_extended_high_seq_num_.end()) {
number_of_packets =
report_block.extended_highest_sequence_number - seq_num_it->second; //反馈最大序列号-上次最大序列号 中间包即为包数
}
fraction_lost_aggregate += number_of_packets * report_block.fraction_lost; //发送数目乘以丢包率等于丢包数目做累加
total_number_of_packets += number_of_packets; //总的发送数目
// Update last received for this SSRC.
ssrc_to_last_received_extended_high_seq_num_[report_block.source_ssrc] =
report_block.extended_highest_sequence_number; //更新当前ssrc反馈的最大seq
}
if (total_number_of_packets < 0) {
//在ssrc_to_last_received_extended_high_seq_num_中找不到则返回忽略
RTC_LOG(LS_WARNING)
<< "Received report block where extended high sequence "
"number goes backwards, ignoring.";
return;
}
if (total_number_of_packets == 0)
fraction_lost_aggregate = 0;
else
fraction_lost_aggregate =
(fraction_lost_aggregate + total_number_of_packets / 2) / //计算占比取整?
total_number_of_packets;
if (fraction_lost_aggregate > 255)
return;
RTC_DCHECK_GE(total_number_of_packets, 0);
//根据丢包估算码率
bandwidth_estimation_.UpdateReceiverBlock(fraction_lost_aggregate, rtt,
total_number_of_packets, now_ms);
}
MaybeTriggerOnNetworkChanged();
}
void SendSideBandwidthEstimation::UpdateReceiverBlock(uint8_t fraction_loss,
int64_t rtt,
int number_of_packets,
int64_t now_ms) {
last_feedback_ms_ = now_ms; //更新最近feedback时间
if (first_report_time_ms_ == -1)
first_report_time_ms_ = now_ms;
// Update RTT if we were able to compute an RTT based on this RTCP.
// FlexFEC doesn't send RTCP SR, which means we won't be able to compute RTT.
if (rtt > 0)
last_round_trip_time_ms_ = rtt; //更新rtt
// Check sequence number diff and weight loss report
if (number_of_packets > 0) { //根据反馈报文统计的发送包数大于0
// Calculate number of lost packets.
const int num_lost_packets_Q8 = fraction_loss * number_of_packets; //丢包数
// Accumulate reports.
lost_packets_since_last_loss_update_Q8_ += num_lost_packets_Q8; //总的累计丢包数
expected_packets_since_last_loss_update_ += number_of_packets; //期望接收的包总数
// Don't generate a loss rate until it can be based on enough packets.
if (expected_packets_since_last_loss_update_ < kLimitNumPackets) // 丢包太小不予以计算
return;
has_decreased_since_last_fraction_loss_ = false; //是否已经降低码率
last_fraction_loss_ = lost_packets_since_last_loss_update_Q8_ /
expected_packets_since_last_loss_update_; //丢包率
// Reset accumulators.
lost_packets_since_last_loss_update_Q8_ = 0;
expected_packets_since_last_loss_update_ = 0;
last_packet_report_ms_ = now_ms; //更新最后一次接收rr报文时间
UpdateEstimate(now_ms);
}
UpdateUmaStats(now_ms, rtt, (fraction_loss * number_of_packets) >> 8); //暂时忽略
}
//根据丢包估算码率
void SendSideBandwidthEstimation::UpdateEstimate(int64_t now_ms) {
uint32_t new_bitrate = current_bitrate_bps_; //新的码率
// We trust the REMB and/or delay-based estimate during the first 2 seconds if
// we haven't had any packet loss reported, to allow startup bitrate probing.
if (last_fraction_loss_ == 0 && IsInStartPhase(now_ms)) {//如果没有丢包 前秒取remb和基于延迟估计得出的码率
new_bitrate = std::max(bwe_incoming_, new_bitrate);
new_bitrate = std::max(delay_based_bitrate_bps_, new_bitrate);
if (new_bitrate != current_bitrate_bps_) {
min_bitrate_history_.clear();
min_bitrate_history_.push_back(
std::make_pair(now_ms, current_bitrate_bps_)); //将当前时间戳 对应的码率存储起来
CapBitrateToThresholds(now_ms, new_bitrate);
return;
}
}
UpdateMinHistory(now_ms); //从min_history中删除老的记录间隔大于1秒?
if (last_packet_report_ms_ == -1) {
// No feedback received.
CapBitrateToThresholds(now_ms, current_bitrate_bps_);
return;
}
int64_t time_since_packet_report_ms = now_ms - last_packet_report_ms_; //这里看代码一直应该一直为0
int64_t time_since_feedback_ms = now_ms - last_feedback_ms_; //从上次反馈到现在的时间
if (time_since_packet_report_ms < 1.2 * kFeedbackIntervalMs) { //在时间窗口内
// We only care about loss above a given bitrate threshold.
float loss = last_fraction_loss_ / 256.0f; //丢包率8位值 计算丢包率
// We only make decisions based on loss when the bitrate is above a
// threshold. This is a crude way of handling loss which is uncorrelated
// to congestion.
if (current_bitrate_bps_ < bitrate_threshold_bps_ || //带宽下限 且丢包<%2
loss <= low_loss_threshold_) {
// Loss < 2%: Increase rate by 8% of the min bitrate in the last
// kBweIncreaseIntervalMs.
// Note that by remembering the bitrate over the last second one can
// rampup up one second faster than if only allowed to start ramping
// at 8% per second rate now. E.g.:
// If sending a constant 100kbps it can rampup immediatly to 108kbps
// whenever a receiver report is received with lower packet loss.
// If instead one would do: current_bitrate_bps_ *= 1.08^(delta time),
// it would take over one second since the lower packet loss to achieve
// 108kbps.
new_bitrate = static_cast<uint32_t>( //从上次基础上增加1.08被并加
min_bitrate_history_.front().second * 1.08 + 0.5);
// Add 1 kbps extra, just to make sure that we do not get stuck
// (gives a little extra increase at low rates, negligible at higher
// rates).
new_bitrate += 1000; //额外增加1kb
} else if (current_bitrate_bps_ > bitrate_threshold_bps_) { //超过码率上限
if (loss <= high_loss_threshold_) { //最大丢包率下 保持不变
// Loss between 2% - 10%: Do nothing.
} else {
// Loss > 10%: Limit the rate decreases to once a kBweDecreaseIntervalMs
// + rtt.
if (!has_decreased_since_last_fraction_loss_ && //丢包率大于10% 且距离上次调节大于最小间隔加一个rtt
(now_ms - time_last_decrease_ms_) >=
(kBweDecreaseIntervalMs + last_round_trip_time_ms_)) {
time_last_decrease_ms_ = now_ms;
// Reduce rate:
// newRate = rate * (1 - 0.5*lossRate);
// where packetLoss = 256*lossRate;
new_bitrate = static_cast<uint32_t>( //按丢包率减小码率
(current_bitrate_bps_ *
static_cast<double>(512 - last_fraction_loss_)) /
512.0);
has_decreased_since_last_fraction_loss_ = true;
}
}
}
} else if (time_since_feedback_ms > //长时间收不到feedback 如果开启了in_timeout_experiment_也会降低码率
kFeedbackTimeoutIntervals * kFeedbackIntervalMs &&
(last_timeout_ms_ == -1 ||
now_ms - last_timeout_ms_ > kTimeoutIntervalMs)) {
if (in_timeout_experiment_) {
RTC_LOG(LS_WARNING) << "Feedback timed out (" << time_since_feedback_ms
<< " ms), reducing bitrate.";
new_bitrate *= 0.8;
// Reset accumulators since we've already acted on missing feedback and
// shouldn't to act again on these old lost packets.
lost_packets_since_last_loss_update_Q8_ = 0;
expected_packets_since_last_loss_update_ = 0;
last_timeout_ms_ = now_ms;
}
}
CapBitrateToThresholds(now_ms, new_bitrate);
}
//综合remb 丢包 延迟估算码率得出当前码率值 (综合取三个得最小值在和配置得最大最小码率去比较)
void SendSideBandwidthEstimation::CapBitrateToThresholds(int64_t now_ms,
uint32_t bitrate_bps) {
if (bwe_incoming_ > 0 && bitrate_bps > bwe_incoming_) { //remb反馈大于0 且丢包大于remb就取小的
bitrate_bps = bwe_incoming_;
}
if (delay_based_bitrate_bps_ > 0 && bitrate_bps > delay_based_bitrate_bps_) {
bitrate_bps = delay_based_bitrate_bps_;
}
if (bitrate_bps > max_bitrate_configured_) { //超过最大 取当前
bitrate_bps = max_bitrate_configured_;
}
if (bitrate_bps < min_bitrate_configured_) { //小于最小 取当前
if (last_low_bitrate_log_ms_ == -1 || //每隔多久打最小带宽日志
now_ms - last_low_bitrate_log_ms_ > kLowBitrateLogPeriodMs) {
RTC_LOG(LS_WARNING) << "Estimated available bandwidth "
<< bitrate_bps / 1000
<< " kbps is below configured min bitrate "
<< min_bitrate_configured_ / 1000 << " kbps.";
last_low_bitrate_log_ms_ = now_ms;
}
bitrate_bps = min_bitrate_configured_;
}
if (bitrate_bps != current_bitrate_bps_ ||
last_fraction_loss_ != last_logged_fraction_loss_ || //变化记录日志
now_ms - last_rtc_event_log_ms_ > kRtcEventLogPeriodMs) {
event_log_->Log(rtc::MakeUnique<RtcEventBweUpdateLossBased>(
bitrate_bps, last_fraction_loss_,
expected_packets_since_last_loss_update_));
last_logged_fraction_loss_ = last_fraction_loss_;
last_rtc_event_log_ms_ = now_ms;
}
current_bitrate_bps_ = bitrate_bps;
}
接收端以一定的频率发送RTCP包(RR、REMB、NACK等)时,会统计两次发送间隔之间(fraction)的接收包信息
两次发送间隔之间理论上应该收到的包数量=当前接收到的最大包序号-上个时刻最大有序包序号 uint16_t exp_since_last = (received_seq_max_ - last_report_seq_max_);
两次发送间隔之间实际接收到有序包的数量=当前时刻收到的有序包的数量-上一个时刻收到的有序包的数量 uint32_t rec_since_last = Count2 - Count1
丢包数=理论上应收的包数-实际收到的包数 int32_t missing = exp_since_last - rec_since_last。
丢包码率计算公式
式中 As(tk) 即为 tk 时刻的带宽估计值,fl(tk)即为 tk 时刻的丢包率,实际代码中我们用到了1.08在爬升时?实际值可以根据实际网络进行调参。
从上面代码种看出得一些疑问:
1.在丢包率不高得时候直接增加1kb得爬升速度是否不合适?
2.丢包区间在2%到10%之间时维持不变?这个是否适合实际环境?是否有相关运维数据支撑?
3.当rtcp包长时间未收到或丢失时,丢包评估机制失效?这个时候是依赖啥来评估码率?
三.基于接收端的码率反馈在发送端的处理
这里暂时不讲述接收端怎么估算得流程,只讲述remb报文到本地后怎么去处理相关逻辑得流程。
逻辑较少 就是传递接收端估算得码率到SendSideBandwidthEstimation中,没有其它特殊处理。
四.基于发送端transport-wide-cc估计发送带宽的码率控制
收rtcp包逻辑和上述一致,这里不予以重复介绍,调用栈为
transport_feedback_observer_->OnTransportFeedback(*packet_information.transport_feedback);
SendSideCongestionController::OnTransportFeedback(const rtcp::TransportFeedback& feedback) ;
DelayBasedBwe::OnDelayBasedBweResult();
bandwidth_estimation_.UpdateDelayBasedEstimate(clock_->TimeInMilliseconds(),
result.target_bitrate_bps);
这里具体细节不展开,主要是讲下调用栈,后续会单独开blog讲解这个流程。
大概先科普下过程 ,基于延迟的拥塞控制算法可以分成以下4个部分:(1)到达时间模型(arrive-time model);(2)预过滤(Pre-filtering);(3)到达时间滤波器(arrive-time filter);(4)过载检测器(over-use detector)
这里有篇文章对应分析了这个简单过程得每个步骤 ,还算简单易懂,https://blog.csdn.net/fishmai/article/details/78793512?utm_medium=distribute.pc_relevant.none-task-blog-baidujs_title-1&spm=1001.2101.3001.4242
五.估算得码率需生效到各个模块得机制
可以发现上述三种码率估算完后都会调用BitrateControllerImpl::MaybeTriggerOnNetworkChanged,而在收到transoirtfeedback时 也会调用MaybeTriggerOnNetworkChanged
来看下2种区别
void BitrateControllerImpl::MaybeTriggerOnNetworkChanged() {
if (!observer_) //在65分支版本实际这里return 了
return;
uint32_t bitrate_bps;
uint8_t fraction_loss;
int64_t rtt;
if (GetNetworkParameters(&bitrate_bps, &fraction_loss, &rtt))
observer_->OnNetworkChanged(bitrate_bps, fraction_loss, rtt);
}
void SendSideCongestionController::OnTransportFeedback(
const rtcp::TransportFeedback& feedback) {
RTC_DCHECK_RUNS_SERIALIZED(&worker_race_);
transport_feedback_adapter_.OnTransportFeedback(feedback);
std::vector<PacketFeedback> feedback_vector = ReceivedPacketFeedbackVector(
transport_feedback_adapter_.GetTransportFeedbackVector());
SortPacketFeedbackVector(&feedback_vector);
bool currently_in_alr =
pacer_->GetApplicationLimitedRegionStartTime().has_value();
if (was_in_alr_ && !currently_in_alr) {
int64_t now_ms = rtc::TimeMillis();
acknowledged_bitrate_estimator_->SetAlrEndedTimeMs(now_ms);
probe_controller_->SetAlrEndedTimeMs(now_ms);
}
was_in_alr_ = currently_in_alr;
acknowledged_bitrate_estimator_->IncomingPacketFeedbackVector(
feedback_vector);
DelayBasedBwe::Result result;
{
rtc::CritScope cs(&bwe_lock_);
result = delay_based_bwe_->IncomingPacketFeedbackVector(
feedback_vector, acknowledged_bitrate_estimator_->bitrate_bps());
}
if (result.updated) {
bitrate_controller_->OnDelayBasedBweResult(result); //而这里进去实际会调应上面
// Update the estimate in the ProbeController, in case we want to probe.
MaybeTriggerOnNetworkChanged(); //这里调用
}
if (result.recovered_from_overuse)
probe_controller_->RequestProbe();
if (in_cwnd_experiment_)
LimitOutstandingBytes(transport_feedback_adapter_.GetOutstandingBytes());
}
void SendSideCongestionController::MaybeTriggerOnNetworkChanged() {
uint32_t bitrate_bps;
uint8_t fraction_loss;
int64_t rtt;
bool estimate_changed = bitrate_controller_->GetNetworkParameters(
&bitrate_bps, &fraction_loss, &rtt);
//RTC_LOG(LS_INFO) << " bps: " << bitrate_bps << " lost " << fraction_loss << " rtt "<< rtt << "\n";
if (estimate_changed) { //这里 可以看到码率会设置进pacer模块 ProbeController模块 和重传retransmission_rate_limiter_模块
pacer_->SetEstimatedBitrate(bitrate_bps);
probe_controller_->SetEstimatedBitrate(bitrate_bps);
retransmission_rate_limiter_->SetMaxRate(bitrate_bps);
}
if (!pacer_pushback_experiment_) {
bitrate_bps = IsNetworkDown() || IsSendQueueFull() ? 0 : bitrate_bps;
} else {
if (IsNetworkDown()) {
bitrate_bps = 0;
} else {
int64_t queue_length_ms = pacer_->ExpectedQueueTimeMs();
if (queue_length_ms == 0) {
encoding_rate_ = 1.0;
} else if (queue_length_ms > 50) {
float encoding_rate = 1.0 - queue_length_ms / 1000.0;
encoding_rate_ = std::min(encoding_rate_, encoding_rate);
encoding_rate_ = std::max(encoding_rate_, 0.0f);
}
bitrate_bps *= encoding_rate_;
bitrate_bps = bitrate_bps < 50000 ? 0 : bitrate_bps;
}
}
if (HasNetworkParametersToReportChanged(bitrate_bps, fraction_loss, rtt)) {
int64_t probing_interval_ms;
{
rtc::CritScope cs(&bwe_lock_);
probing_interval_ms = delay_based_bwe_->GetExpectedBwePeriodMs();
}
{
rtc::CritScope cs(&observer_lock_);
if (observer_) {
observer_->OnNetworkChanged(bitrate_bps, fraction_loss, rtt, //这里还会通知observer,这个observer是什么,调试进入call模块
probing_interval_ms);
}
}
}
}
//码率发生变化时会通知这里
void Call::OnNetworkChanged(uint32_t target_bitrate_bps,
uint8_t fraction_loss,
int64_t rtt_ms,
int64_t probing_interval_ms) {
// TODO(perkj): Consider making sure CongestionController operates on
// |worker_queue_|.
if (!worker_queue_.IsCurrent()) {
worker_queue_.PostTask(
[this, target_bitrate_bps, fraction_loss, rtt_ms, probing_interval_ms] {
OnNetworkChanged(target_bitrate_bps, fraction_loss, rtt_ms,
probing_interval_ms);
});
return;
}
RTC_DCHECK_RUN_ON(&worker_queue_);
// For controlling the rate of feedback messages.
receive_side_cc_.OnBitrateChanged(target_bitrate_bps); //主要看这2个
bitrate_allocator_->OnNetworkChanged(target_bitrate_bps, fraction_loss,
rtt_ms, probing_interval_ms);
// Ignore updates if bitrate is zero (the aggregate network state is down).
if (target_bitrate_bps == 0) {
rtc::CritScope lock(&bitrate_crit_);
estimated_send_bitrate_kbps_counter_.ProcessAndPause();
pacer_bitrate_kbps_counter_.ProcessAndPause();
return;
}
bool sending_video;
{
ReadLockScoped read_lock(*send_crit_);
sending_video = !video_send_streams_.empty();
}
rtc::CritScope lock(&bitrate_crit_);
if (!sending_video) {
// Do not update the stats if we are not sending video.
estimated_send_bitrate_kbps_counter_.ProcessAndPause();
pacer_bitrate_kbps_counter_.ProcessAndPause();
return;
}
estimated_send_bitrate_kbps_counter_.Add(target_bitrate_bps / 1000);
// Pacer bitrate may be higher than bitrate estimate if enforcing min bitrate.
uint32_t pacer_bitrate_bps =
std::max(target_bitrate_bps, min_allocated_send_bitrate_bps_);
pacer_bitrate_kbps_counter_.Add(pacer_bitrate_bps / 1000);
}
从调试种可以得出 在开启cc得情况下实际生效得只是在 SendSideCongestionController::MaybeTriggerOnNetworkChanged()中,而通过上面 会回调call::OnNetworkChanged 最终进入BitrateAllocator::OnNetworkChanged,接下来看下这段代码得逻辑,里面会将具体得码率设置到不同模块,待研究
void BitrateAllocator::OnNetworkChanged(uint32_t target_bitrate_bps,
uint8_t fraction_loss,
int64_t rtt,
int64_t bwe_period_ms) {
RTC_DCHECK_CALLED_SEQUENTIALLY(&sequenced_checker_);
last_bitrate_bps_ = target_bitrate_bps;
last_non_zero_bitrate_bps_ =
target_bitrate_bps > 0 ? target_bitrate_bps : last_non_zero_bitrate_bps_;
last_fraction_loss_ = fraction_loss;
last_rtt_ = rtt;
last_bwe_period_ms_ = bwe_period_ms;
// Periodically log the incoming BWE.
int64_t now = clock_->TimeInMilliseconds();
if (now > last_bwe_log_time_ + kBweLogIntervalMs) {
RTC_LOG(LS_INFO) << "Current BWE " << target_bitrate_bps;
last_bwe_log_time_ = now;
}
ObserverAllocation allocation = AllocateBitrates(target_bitrate_bps);
for (auto& config : bitrate_observer_configs_) {
uint32_t allocated_bitrate = allocation[config.observer];
uint32_t protection_bitrate = config.observer->OnBitrateUpdated(
allocated_bitrate, last_fraction_loss_, last_rtt_,
last_bwe_period_ms_);
if (allocated_bitrate == 0 && config.allocated_bitrate_bps > 0) {
if (target_bitrate_bps > 0)
++num_pause_events_;
// The protection bitrate is an estimate based on the ratio between media
// and protection used before this observer was muted.
uint32_t predicted_protection_bps =
(1.0 - config.media_ratio) * config.min_bitrate_bps;
RTC_LOG(LS_INFO) << "Pausing observer " << config.observer
<< " with configured min bitrate "
<< config.min_bitrate_bps << " and current estimate of "
<< target_bitrate_bps << " and protection bitrate "
<< predicted_protection_bps;
} else if (allocated_bitrate > 0 && config.allocated_bitrate_bps == 0) {
if (target_bitrate_bps > 0)
++num_pause_events_;
RTC_LOG(LS_INFO) << "Resuming observer " << config.observer
<< ", configured min bitrate " << config.min_bitrate_bps
<< ", current allocation " << allocated_bitrate
<< " and protection bitrate " << protection_bitrate;
}
// Only update the media ratio if the observer got an allocation.
if (allocated_bitrate > 0)
config.media_ratio = MediaRatio(allocated_bitrate, protection_bitrate);
config.allocated_bitrate_bps = allocated_bitrate;
}
UpdateAllocationLimits();
}
思考:
1、webrtc的发送带宽估计是针对每一路流还是总的带宽
答案:总的带宽
2、webrtc的remb是统计的整体带宽吗?
答案:是的
3、如果webrtc同时观看了多路流,如何针对每一路流反馈带宽,丢包等信息
答案:transport-wide-cc
4、如果webrtc同时发送了多路流,如何估计每一路的带宽情况
答案:同上
5. 估算的码率如何作用于各个模块的机制
答案:看上文