1 几个定义
class RtpVideoStreamReceiver : public LossNotificationSender,
public RecoveredPacketReceiver,
public RtpPacketSinkInterface,
public KeyFrameRequestSender,
public video_coding::OnAssembledFrameCallback,
public video_coding::OnCompleteFrameCallback,
public OnDecryptedFrameCallback,
public OnDecryptionStatusChangeCallback
===>
RemoteNtpTimeEstimator ntp_estimator_; //
class RemoteNtpTimeEstimator {
public:
explicit RemoteNtpTimeEstimator(Clock* clock);
~RemoteNtpTimeEstimator();
// Updates the estimator with round trip time |rtt|, NTP seconds |ntp_secs|,
// NTP fraction |ntp_frac| and RTP timestamp |rtcp_timestamp|.
bool UpdateRtcpTimestamp(int64_t rtt,
uint32_t ntp_secs,
uint32_t ntp_frac,
uint32_t rtp_timestamp);
// Estimates the NTP timestamp in local timebase from |rtp_timestamp|.
// Returns the NTP timestamp in ms when success. -1 if failed.
int64_t Estimate(uint32_t rtp_timestamp);
private:
Clock* clock_;
MovingMedianFilter<int64_t> ntp_clocks_offset_estimator_;
RtpToNtpEstimator rtp_to_ntp_; //
int64_t last_timing_log_ms_;
RTC_DISALLOW_COPY_AND_ASSIGN(RemoteNtpTimeEstimator);
};
class RtpToNtpEstimator {
public:
RtpToNtpEstimator();
~RtpToNtpEstimator();
// RTP and NTP timestamp pair from a RTCP SR report.
struct RtcpMeasurement {
RtcpMeasurement(uint32_t ntp_secs,
uint32_t ntp_frac,
int64_t unwrapped_timestamp);
bool IsEqual(const RtcpMeasurement& other) const;
NtpTime ntp_time;
int64_t unwrapped_rtp_timestamp;
};
// Estimated parameters from RTP and NTP timestamp pairs in |measurements_|.
struct Parameters {
Parameters() : frequency_khz(0.0), offset_ms(0.0) {}
Parameters(double frequency_khz, double offset_ms)
: frequency_khz(frequency_khz), offset_ms(offset_ms) {}
double frequency_khz;
double offset_ms;
};
// Updates measurements with RTP/NTP timestamp pair from a RTCP sender report.
// |new_rtcp_sr| is set to true if a new report is added.
bool UpdateMeasurements(uint32_t ntp_secs,
uint32_t ntp_frac,
uint32_t rtp_timestamp,
bool* new_rtcp_sr);
// Converts an RTP timestamp to the NTP domain in milliseconds.
// Returns true on success, false otherwise.
bool Estimate(int64_t rtp_timestamp, int64_t* ntp_timestamp_ms) const;
// Returns estimated rtp to ntp linear transform parameters.
const absl::optional<Parameters> params() const;
static const int kMaxInvalidSamples = 3;
private:
void UpdateParameters(); //
int consecutive_invalid_samples_;
std::list<RtcpMeasurement> measurements_;
absl::optional<Parameters> params_; //
mutable TimestampUnwrapper unwrapper_;
};2 线性回归过程
PacketReceiver::DeliveryStatus Call::DeliverPacket(
MediaType media_type,
rtc::CopyOnWriteBuffer packet,
int64_t packet_time_us) {
RTC_DCHECK_RUN_ON(&configuration_sequence_checker_);
if (IsRtcp(packet.cdata(), packet.size()))
return DeliverRtcp(media_type, packet.cdata(), packet.size()); // 处理 RTCP 报文
return DeliverRtp(media_type, std::move(packet), packet_time_us);
}
// 通过缩进来表示一些中间对象的创建过程
VideoReceiveStream::VideoReceiveStream(
TaskQueueFactory* task_queue_factory,
RtpStreamReceiverControllerInterface* receiver_controller,
int num_cpu_cores,
PacketRouter* packet_router,
VideoReceiveStream::Config config, // config 来自 WebRtcVideoChannel::AddRecvStream
ProcessThread* process_thread,
CallStats* call_stats,
Clock* clock,
VCMTiming* timing)
: task_queue_factory_(task_queue_factory),
transport_adapter_(config.rtcp_send_transport), // config.rtcp_send_transport 就是 WebRtcVideoChannel 的 this指针，具体见 WebRtcVideoChannel::AddRecvStream
config_(std::move(config)), // transport_adapter_ 是 webrtc::internal::TransportAdapter 类型，其内包含 webrtc::Transport* 的成员变量，该变量指向 WebRtcVideoChannel
num_cpu_cores_(num_cpu_cores),
process_thread_(process_thread),
clock_(clock),
call_stats_(call_stats),
source_tracker_(clock_),
stats_proxy_(&config_, clock_),
rtp_receive_statistics_(ReceiveStatistics::Create(clock_)),
timing_(timing),
video_receiver_(clock_, timing_.get()),
rtp_video_stream_receiver_(clock_,
&transport_adapter_, //
call_stats,
packet_router,
&config_,
rtp_receive_statistics_.get(),
&stats_proxy_,
process_thread_,
this, // NackSender
nullptr, // Use default KeyFrameRequestSender
this, // OnCompleteFrameCallback
config_.frame_decryptor), // RtpVideoStreamReceiver rtp_video_stream_receiver_;
RtpVideoStreamReceiver::RtpVideoStreamReceiver(
Clock* clock,
Transport* transport, // transport 指向的是 webrtc::internal::TransportAdapter, 其内包含 webrtc::Transport* 的成员变量，该变量指向 WebRtcVideoChannel
RtcpRttStats* rtt_stats,
PacketRouter* packet_router,
const VideoReceiveStream::Config* config,
ReceiveStatistics* rtp_receive_statistics,
ReceiveStatisticsProxy* receive_stats_proxy,
ProcessThread* process_thread,
NackSender* nack_sender,
KeyFrameRequestSender* keyframe_request_sender,
video_coding::OnCompleteFrameCallback* complete_frame_callback,
rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor)
: clock_(clock),
config_(*config),
packet_router_(packet_router),
process_thread_(process_thread),
ntp_estimator_(clock),
rtp_header_extensions_(config_.rtp.extensions),
rtp_receive_statistics_(rtp_receive_statistics),
ulpfec_receiver_(UlpfecReceiver::Create(config->rtp.remote_ssrc,
this,
config->rtp.extensions)),
receiving_(false),
last_packet_log_ms_(-1),
rtp_rtcp_(CreateRtpRtcpModule(clock,
rtp_receive_statistics_,
transport, // transport 指向的是 webrtc::internal::TransportAdapter, 其内包含 webrtc::Transport* 的成员变量，该变量指向 WebRtcVideoChannel
rtt_stats,
receive_stats_proxy,
config_.rtp.local_ssrc)), // const std::unique_ptr<RtpRtcp> rtp_rtcp_;
complete_frame_callback_(complete_frame_callback),
keyframe_request_sender_(keyframe_request_sender),
// TODO(bugs.webrtc.org/10336): Let |rtcp_feedback_buffer_| communicate
// directly with |rtp_rtcp_|.
rtcp_feedback_buffer_(this, nack_sender, this),
packet_buffer_(clock_,
kPacketBufferStartSize,
PacketBufferMaxSize(),
this),
has_received_frame_(false),
frames_decryptable_(false) {
constexpr bool remb_candidate = true;
if (packet_router_)
packet_router_->AddReceiveRtpModule(rtp_rtcp_.get(), remb_candidate);
RTC_DCHECK(config_.rtp.rtcp_mode != RtcpMode::kOff)
<< "A stream should not be configured with RTCP disabled. This value is "
"reserved for internal usage.";
// TODO(pbos): What's an appropriate local_ssrc for receive-only streams?
RTC_DCHECK(config_.rtp.local_ssrc != 0);
RTC_DCHECK(config_.rtp.remote_ssrc != config_.rtp.local_ssrc);
rtp_rtcp_->SetRTCPStatus(config_.rtp.rtcp_mode);
rtp_rtcp_->SetRemoteSSRC(config_.rtp.remote_ssrc);
static const int kMaxPacketAgeToNack = 450;
const int max_reordering_threshold = (config_.rtp.nack.rtp_history_ms > 0)
? kMaxPacketAgeToNack
: kDefaultMaxReorderingThreshold;
rtp_receive_statistics_->SetMaxReorderingThreshold(config_.rtp.remote_ssrc,
max_reordering_threshold);
// TODO(nisse): For historic reasons, we applied the above
// max_reordering_threshold also for RTX stats, which makes little sense since
// we don't NACK rtx packets. Consider deleting the below block, and rely on
// the default threshold.
if (config_.rtp.rtx_ssrc) {
rtp_receive_statistics_->SetMaxReorderingThreshold(
config_.rtp.rtx_ssrc, max_reordering_threshold);
}
if (config_.rtp.rtcp_xr.receiver_reference_time_report)
rtp_rtcp_->SetRtcpXrRrtrStatus(true);
// Stats callback for CNAME changes.
rtp_rtcp_->RegisterRtcpCnameCallback(receive_stats_proxy);
process_thread_->RegisterModule(rtp_rtcp_.get(), RTC_FROM_HERE);
if (config_.rtp.lntf.enabled) {
loss_notification_controller_ =
std::make_unique<LossNotificationController>(&rtcp_feedback_buffer_,
&rtcp_feedback_buffer_);
}
if (config_.rtp.nack.rtp_history_ms != 0) {
nack_module_ = std::make_unique<NackModule>(clock_, &rtcp_feedback_buffer_,
&rtcp_feedback_buffer_);
process_thread_->RegisterModule(nack_module_.get(), RTC_FROM_HERE);
}
reference_finder_ =
std::make_unique<video_coding::RtpFrameReferenceFinder>(this);
// Only construct the encrypted receiver if frame encryption is enabled.
if (config_.crypto_options.sframe.require_frame_encryption) {
buffered_frame_decryptor_ =
std::make_unique<BufferedFrameDecryptor>(this, this);
if (frame_decryptor != nullptr) {
buffered_frame_decryptor_->SetFrameDecryptor(std::move(frame_decryptor));
}
}
}
std::unique_ptr<RtpRtcp> CreateRtpRtcpModule(
Clock* clock,
ReceiveStatistics* receive_statistics,
Transport* outgoing_transport, // 注意这里
RtcpRttStats* rtt_stats,
RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer,
uint32_t local_ssrc) {
RtpRtcp::Configuration configuration;
configuration.clock = clock;
configuration.audio = false;
configuration.receiver_only = true;
configuration.receive_statistics = receive_statistics;
configuration.outgoing_transport = outgoing_transport; //注意这里
configuration.rtt_stats = rtt_stats;
configuration.rtcp_packet_type_counter_observer =
rtcp_packet_type_counter_observer;
configuration.local_media_ssrc = local_ssrc;
std::unique_ptr<RtpRtcp> rtp_rtcp = RtpRtcp::Create(configuration); // 返回的是指向 ModuleRtpRtcpImpl 的智能指针
rtp_rtcp->SetRTCPStatus(RtcpMode::kCompound);
return rtp_rtcp;
}
// ModuleRtpRtcpImpl 的派生关系
class RtpRtcp : public Module, public RtcpFeedbackSenderInterface
class ModuleRtpRtcpImpl : public RtpRtcp, public RTCPReceiver::ModuleRtpRtcp
std::unique_ptr<RtpRtcp> RtpRtcp::Create(const Configuration& configuration) {
RTC_DCHECK(configuration.clock);
return std::make_unique<ModuleRtpRtcpImpl>(configuration); //注意这里
}
bool RtpVideoStreamReceiver::DeliverRtcp(const uint8_t* rtcp_packet,
size_t rtcp_packet_length) {
RTC_DCHECK_RUN_ON(&worker_task_checker_);
if (!receiving_) {
return false;
}
rtp_rtcp_->IncomingRtcpPacket(rtcp_packet, rtcp_packet_length); // ModuleRtpRtcpImpl::IncomingRtcpPacket
int64_t rtt = 0;
rtp_rtcp_->RTT(config_.rtp.remote_ssrc, &rtt, nullptr, nullptr, nullptr); // ModuleRtpRtcpImpl::RTT
if (rtt == 0) {
// Waiting for valid rtt.
return true;
}
uint32_t ntp_secs = 0;
uint32_t ntp_frac = 0;
uint32_t rtp_timestamp = 0;
uint32_t recieved_ntp_secs = 0;
uint32_t recieved_ntp_frac = 0;
// ntp_secs 就是最近一次接收到的SR包中的NTP时间中的单位为秒的部分
// ntp_frac 就是最近一次接收到的SR包中的NTP时间中的单位为1/2^32秒的部分
// recieved_ntp_secs 表示接收到这个SR包的时候，本地时间的NTP表示中的单位为秒的部分
// recieved_ntp_frac 表示接收到这个SR包的时候，本地时间的NTP表示中的单位为1/2^32秒的部分
// rtp_timestamp 表示最近一次接收到的SR包中的 RTP 时间戳
if (rtp_rtcp_->RemoteNTP(&ntp_secs, &ntp_frac, &recieved_ntp_secs,
&recieved_ntp_frac, &rtp_timestamp) != 0) { // ModuleRtpRtcpImpl::RemoteNTP
// Waiting for RTCP.
return true;
}
// 接收到最近一次SR包的本地时间的NTP表示，由于发送端和接收端没有进行时间同步，所以SR包中的NTP时间和本地的NTP时间很有可能有较大的的偏差
NtpTime recieved_ntp(recieved_ntp_secs, recieved_ntp_frac);
// 从最近一次收到SR包到现在所流逝的时间
int64_t time_since_recieved =
clock_->CurrentNtpInMilliseconds() - recieved_ntp.ToMs();
// Don't use old SRs to estimate time.
if (time_since_recieved <= 1) {
ntp_estimator_.UpdateRtcpTimestamp(rtt, ntp_secs, ntp_frac, rtp_timestamp); // RemoteNtpTimeEstimator::UpdateRtcpTimestamp
}
return true;
}
// 接收 RTCP 包
void ModuleRtpRtcpImpl::IncomingRtcpPacket(const uint8_t* rtcp_packet,
const size_t length) {
rtcp_receiver_.IncomingPacket(rtcp_packet, length); // RTCPReceiver::IncomingPacket
}
void RTCPReceiver::IncomingPacket(const uint8_t* packet, size_t packet_size) {
if (packet_size == 0) {
RTC_LOG(LS_WARNING) << "Incoming empty RTCP packet";
return;
}
PacketInformation packet_information;
if (!ParseCompoundPacket(packet, packet + packet_size, &packet_information))
return;
TriggerCallbacksFromRtcpPacket(packet_information);
}
// 计算 RTT
int32_t ModuleRtpRtcpImpl::RTT(const uint32_t remote_ssrc,
int64_t* rtt,
int64_t* avg_rtt,
int64_t* min_rtt,
int64_t* max_rtt) const {
int32_t ret = rtcp_receiver_.RTT(remote_ssrc, rtt, avg_rtt, min_rtt, max_rtt); // RTCPReceiver::RTT
if (rtt && *rtt == 0) {
// Try to get RTT from RtcpRttStats class.
*rtt = rtt_ms();
}
return ret;
}
int32_t RTCPReceiver::RTT(uint32_t remote_ssrc,
int64_t* last_rtt_ms,
int64_t* avg_rtt_ms,
int64_t* min_rtt_ms,
int64_t* max_rtt_ms) const {
rtc::CritScope lock(&rtcp_receiver_lock_);
auto it = received_report_blocks_.find(main_ssrc_);
if (it == received_report_blocks_.end())
return -1;
auto it_info = it->second.find(remote_ssrc);
if (it_info == it->second.end())
return -1;
const ReportBlockData* report_block_data = &it_info->second;
if (report_block_data->num_rtts() == 0)
return -1;
if (last_rtt_ms)
*last_rtt_ms = report_block_data->last_rtt_ms();
if (avg_rtt_ms) {
*avg_rtt_ms =
report_block_data->sum_rtt_ms() / report_block_data->num_rtts();
}
if (min_rtt_ms)
*min_rtt_ms = report_block_data->min_rtt_ms();
if (max_rtt_ms)
*max_rtt_ms = report_block_data->max_rtt_ms();
return 0;
}
// ntp 相关计算
int32_t ModuleRtpRtcpImpl::RemoteNTP(uint32_t* received_ntpsecs,
uint32_t* received_ntpfrac,
uint32_t* rtcp_arrival_time_secs,
uint32_t* rtcp_arrival_time_frac,
uint32_t* rtcp_timestamp) const {
return rtcp_receiver_.NTP(received_ntpsecs, received_ntpfrac, // RTCPReceiver::NTP
rtcp_arrival_time_secs, rtcp_arrival_time_frac,
rtcp_timestamp)
? 0
: -1;
}
bool RTCPReceiver::NTP(uint32_t* received_ntp_secs,
uint32_t* received_ntp_frac,
uint32_t* rtcp_arrival_time_secs,
uint32_t* rtcp_arrival_time_frac,
uint32_t* rtcp_timestamp) const {
rtc::CritScope lock(&rtcp_receiver_lock_);
if (!last_received_sr_ntp_.Valid())
return false;
// NTP from incoming SenderReport.
if (received_ntp_secs)
*received_ntp_secs = remote_sender_ntp_time_.seconds();
if (received_ntp_frac)
*received_ntp_frac = remote_sender_ntp_time_.fractions();
// Rtp time from incoming SenderReport.
if (rtcp_timestamp)
*rtcp_timestamp = remote_sender_rtp_time_;
// Local NTP time when we received a RTCP packet with a send block.
if (rtcp_arrival_time_secs)
*rtcp_arrival_time_secs = last_received_sr_ntp_.seconds();
if (rtcp_arrival_time_frac)
*rtcp_arrival_time_frac = last_received_sr_ntp_.fractions();
return true;
}
// rtt 是在发送端计算出来的RTT
// ntp_secs 就是最近一次接收到的SR包中的NTP时间中的单位为秒的部分
// ntp_frac 就是最近一次接收到的SR包中的NTP时间中的单位为1/2^32秒的部分
// rtp_timestamp 表示最近一次接收到的SR包中的 RTP 时间戳
bool RemoteNtpTimeEstimator::UpdateRtcpTimestamp(int64_t rtt,
uint32_t ntp_secs,
uint32_t ntp_frac,
uint32_t rtcp_timestamp) {
bool new_rtcp_sr = false;
if (!rtp_to_ntp_.UpdateMeasurements(ntp_secs, ntp_frac, rtcp_timestamp, // RtpToNtpEstimator::UpdateMeasurements
&new_rtcp_sr)) {
return false;
}
if (!new_rtcp_sr) {
// No new RTCP SR since last time this function was called.
return true;
}
// Update extrapolator with the new arrival time.
// The extrapolator assumes the TimeInMilliseconds time.
int64_t receiver_arrival_time_ms = clock_->TimeInMilliseconds();
int64_t sender_send_time_ms = Clock::NtpToMs(ntp_secs, ntp_frac);
int64_t sender_arrival_time_ms = sender_send_time_ms + rtt / 2;
int64_t remote_to_local_clocks_offset =
receiver_arrival_time_ms - sender_arrival_time_ms;
ntp_clocks_offset_estimator_.Insert(remote_to_local_clocks_offset);
return true;
}
// ntp_secs 就是最近一次接收到的SR包中的NTP时间中的单位为秒的部分
// ntp_frac 就是最近一次接收到的SR包中的NTP时间中的单位为1/2^32秒的部分
// rtp_timestamp 表示最近一次接收到的SR包中的 RTP
// 注意：发送SR包的肯定是发送者，但如果SR包中也有report block 就证明这个发送者也是接收者，
// 与这个接收者对应的发送者端就可以根据SR包中的report block计算RTT
bool RtpToNtpEstimator::UpdateMeasurements(uint32_t ntp_secs,
uint32_t ntp_frac,
uint32_t rtp_timestamp,
bool* new_rtcp_sr) {
*new_rtcp_sr = false;
int64_t unwrapped_rtp_timestamp = unwrapper_.Unwrap(rtp_timestamp);
// 构建一个度量信息，分析 unwrapper_ 的代码，可以知道 unwrapped_rtp_timestamp 就是不回绕的时间戳。
// 举个例子，回绕的时间戳是：(2^32 - 3600) -> 0 -> 3600，不回绕的时间戳就是：(2^32 - 3600) -> 2^32 -> (2^32 + 3600)
RtcpMeasurement new_measurement(ntp_secs, ntp_frac, unwrapped_rtp_timestamp);
if (Contains(measurements_, new_measurement)) {
// RTCP SR report already added.
return true;
}
if (!new_measurement.ntp_time.Valid())
return false;
int64_t ntp_ms_new = new_measurement.ntp_time.ToMs();
bool invalid_sample = false;
if (!measurements_.empty()) {
int64_t old_rtp_timestamp = measurements_.front().unwrapped_rtp_timestamp;
int64_t old_ntp_ms = measurements_.front().ntp_time.ToMs();
if (ntp_ms_new <= old_ntp_ms ||
ntp_ms_new > old_ntp_ms + kMaxAllowedRtcpNtpIntervalMs) {
invalid_sample = true;
} else if (unwrapped_rtp_timestamp <= old_rtp_timestamp) {
RTC_LOG(LS_WARNING)
<< "Newer RTCP SR report with older RTP timestamp, dropping";
invalid_sample = true;
} else if (unwrapped_rtp_timestamp - old_rtp_timestamp > (1 << 25)) {
// Sanity check. No jumps too far into the future in rtp.
invalid_sample = true;
}
}
if (invalid_sample) {
++consecutive_invalid_samples_;
if (consecutive_invalid_samples_ < kMaxInvalidSamples) {
return false;
}
RTC_LOG(LS_WARNING) << "Multiple consecutively invalid RTCP SR reports, "
"clearing measurements.";
measurements_.clear();
params_ = absl::nullopt;
}
consecutive_invalid_samples_ = 0;
// Insert new RTCP SR report.
if (measurements_.size() == kNumRtcpReportsToUse)
measurements_.pop_back();
// 将度量信息进行保存
measurements_.push_front(new_measurement);
*new_rtcp_sr = true;
// List updated, calculate new parameters.
UpdateParameters(); //
return true;
}
void RtpToNtpEstimator::UpdateParameters() {
if (measurements_.size() < 2)
return;
std::vector<double> x;
std::vector<double> y;
x.reserve(measurements_.size());
y.reserve(measurements_.size());
for (auto it = measurements_.begin(); it != measurements_.end(); ++it) {
x.push_back(it->unwrapped_rtp_timestamp);
y.push_back(it->ntp_time.ToMs());
}
double slope, offset;
// 线性回归
if (!LinearRegression(x, y, &slope, &offset)) {
return;
}
// 保存斜率和截距
params_.emplace(1 / slope, offset);
}
3 相关类图
主要看上图的右上角。
扫一扫
734
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
