1 函数调用关系图
2 代码
void UDPPort::OnReadPacket(rtc::AsyncPacketSocket* socket,
const char* data,
size_t size,
const rtc::SocketAddress& remote_addr,
const int64_t& packet_time_us) {
RTC_DCHECK(socket == socket_);
RTC_DCHECK(!remote_addr.IsUnresolvedIP());
// Look for a response from the STUN server.
// Even if the response doesn't match one of our outstanding requests, we
// will eat it because it might be a response to a retransmitted packet, and
// we already cleared the request when we got the first response.
if (server_addresses_.find(remote_addr) != server_addresses_.end()) {
requests_.CheckResponse(data, size);
return;
}
if (Connection* conn = GetConnection(remote_addr)) {
conn->OnReadPacket(data, size, packet_time_us); //注意这里
} else {
Port::OnReadPacket(data, size, remote_addr, PROTO_UDP);
}
}
void Connection::OnReadPacket(const char* data,
size_t size,
int64_t packet_time_us) {
std::unique_ptr<IceMessage> msg;
std::string remote_ufrag;
const rtc::SocketAddress& addr(remote_candidate_.address());
if (!port_->GetStunMessage(data, size, addr, &msg, &remote_ufrag)) {
// The packet did not parse as a valid STUN message
// This is a data packet, pass it along.
last_data_received_ = rtc::TimeMillis();
UpdateReceiving(last_data_received_);
recv_rate_tracker_.AddSamples(size);
SignalReadPacket(this, data, size, packet_time_us); //注意这里P2PTransportChannel::AddConnection ===>其内 connection->SignalReadPacket.connect(this, &P2PTransportChannel::OnReadPacket); //注意这里
// If timed out sending writability checks, start up again
if (!pruned_ && (write_state_ == STATE_WRITE_TIMEOUT)) {
RTC_LOG(LS_WARNING)
<< "Received a data packet on a timed-out Connection. "
"Resetting state to STATE_WRITE_INIT.";
set_write_state(STATE_WRITE_INIT);
}
} else if (!msg) {
// The packet was STUN, but failed a check and was handled internally.
} else {
// The packet is STUN and passed the Port checks.
// Perform our own checks to ensure this packet is valid.
// If this is a STUN request, then update the receiving bit and respond.
// If this is a STUN response, then update the writable bit.
// Log at LS_INFO if we receive a ping on an unwritable connection.
rtc::LoggingSeverity sev = (!writable() ? rtc::LS_INFO : rtc::LS_VERBOSE);
switch (msg->type()) {
case STUN_BINDING_REQUEST:
RTC_LOG_V(sev) << ToString() << ": Received STUN ping, id="
<< rtc::hex_encode(msg->transaction_id());
if (remote_ufrag == remote_candidate_.username()) {
HandleBindingRequest(msg.get());
} else {
// The packet had the right local username, but the remote username
// was not the right one for the remote address.
RTC_LOG(LS_ERROR)
<< ToString()
<< ": Received STUN request with bad remote username "
<< remote_ufrag;
port_->SendBindingErrorResponse(msg.get(), addr,
STUN_ERROR_UNAUTHORIZED,
STUN_ERROR_REASON_UNAUTHORIZED);
}
break;
// Response from remote peer. Does it match request sent?
// This doesn't just check, it makes callbacks if transaction
// id's match.
case STUN_BINDING_RESPONSE:
case STUN_BINDING_ERROR_RESPONSE:
if (msg->ValidateMessageIntegrity(data, size,
remote_candidate().password())) {
requests_.CheckResponse(msg.get());
}
// Otherwise silently discard the response message.
break;
// Remote end point sent an STUN indication instead of regular binding
// request. In this case |last_ping_received_| will be updated but no
// response will be sent.
case STUN_BINDING_INDICATION:
ReceivedPing(msg->transaction_id());
break;
default:
RTC_NOTREACHED();
break;
}
}
}
void P2PTransportChannel::OnReadPacket(Connection* connection,
const char* data,
size_t len,
int64_t packet_time_us) {
RTC_DCHECK_RUN_ON(network_thread_);
// Do not deliver, if packet doesn't belong to the correct transport channel.
if (!FindConnection(connection))
return;
// Let the client know of an incoming packet
SignalReadPacket(this, data, len, packet_time_us, 0); //注意这里DtlsTransport::ConnectToIceTransport ===> ice_transport_->SignalReadPacket.connect(this, &DtlsTransport::OnReadPacket); //注意这里
// May need to switch the sending connection based on the receiving media path
// if this is the controlled side.
if (ice_role_ == ICEROLE_CONTROLLED) {
MaybeSwitchSelectedConnection(connection, "data received");
}
}
void DtlsTransport::OnReadPacket(rtc::PacketTransportInternal* transport,
const char* data,
size_t size,
const int64_t& packet_time_us,
int flags) {
RTC_DCHECK_RUN_ON(&thread_checker_);
RTC_DCHECK(transport == ice_transport_);
RTC_DCHECK(flags == 0);
if (!dtls_active_) {
// Not doing DTLS.
SignalReadPacket(this, data, size, packet_time_us, 0);
return;
}
switch (dtls_state()) {
case DTLS_TRANSPORT_NEW:
if (dtls_) {
RTC_LOG(LS_INFO) << ToString()
<< ": Packet received before DTLS started.";
} else {
RTC_LOG(LS_WARNING) << ToString()
<< ": Packet received before we know if we are "
"doing DTLS or not.";
}
// Cache a client hello packet received before DTLS has actually started.
if (IsDtlsClientHelloPacket(data, size)) {
RTC_LOG(LS_INFO) << ToString()
<< ": Caching DTLS ClientHello packet until DTLS is "
"started.";
cached_client_hello_.SetData(data, size);
// If we haven't started setting up DTLS yet (because we don't have a
// remote fingerprint/role), we can use the client hello as a clue that
// the peer has chosen the client role, and proceed with the handshake.
// The fingerprint will be verified when it's set.
if (!dtls_ && local_certificate_) {
SetDtlsRole(rtc::SSL_SERVER);
SetupDtls();
}
} else {
RTC_LOG(LS_INFO) << ToString()
<< ": Not a DTLS ClientHello packet; dropping.";
}
break;
case DTLS_TRANSPORT_CONNECTING:
case DTLS_TRANSPORT_CONNECTED:
// We should only get DTLS or SRTP packets; STUN's already been demuxed.
// Is this potentially a DTLS packet?
if (IsDtlsPacket(data, size)) {
if (!HandleDtlsPacket(data, size)) {
RTC_LOG(LS_ERROR) << ToString() << ": Failed to handle DTLS packet.";
return;
}
} else {
// Not a DTLS packet; our handshake should be complete by now.
if (dtls_state() != DTLS_TRANSPORT_CONNECTED) {
RTC_LOG(LS_ERROR) << ToString()
<< ": Received non-DTLS packet before DTLS "
"complete.";
return;
}
// And it had better be a SRTP packet.
if (!IsRtpPacket(data, size)) {
RTC_LOG(LS_ERROR)
<< ToString() << ": Received unexpected non-DTLS packet.";
return;
}
// Sanity check.
RTC_DCHECK(!srtp_ciphers_.empty());
// Signal this upwards as a bypass packet.
SignalReadPacket(this, data, size, packet_time_us, PF_SRTP_BYPASS); // RtpTransport::SetRtpPacketTransport ===> new_packet_transport->SignalReadPacket.connect(this, &RtpTransport::OnReadPacket); //注意这里
} // 此时的 this 指针实际上指向的就是 webrtc::DtlsSrtpTransport
break;
case DTLS_TRANSPORT_FAILED:
case DTLS_TRANSPORT_CLOSED:
// This shouldn't be happening. Drop the packet.
break;
}
}
// webrtc::DtlsSrtpTransport 的继承关系
class webrtc::RtpTransport : public webrtc::RtpTransportInternal
class webrtc::SrtpTransport : public webrtc::RtpTransport
class webrtc::DtlsSrtpTransport : public webrtc::SrtpTransport
void RtpTransport::OnReadPacket(rtc::PacketTransportInternal* transport,
const char* data,
size_t len,
const int64_t& packet_time_us,
int flags) {
TRACE_EVENT0("webrtc", "RtpTransport::OnReadPacket");
// When using RTCP multiplexing we might get RTCP packets on the RTP
// transport. We check the RTP payload type to determine if it is RTCP.
auto array_view = rtc::MakeArrayView(data, len);
cricket::RtpPacketType packet_type = cricket::InferRtpPacketType(array_view);
// Filter out the packet that is neither RTP nor RTCP.
if (packet_type == cricket::RtpPacketType::kUnknown) {
return;
}
// Protect ourselves against crazy data.
if (!cricket::IsValidRtpPacketSize(packet_type, len)) {
RTC_LOG(LS_ERROR) << "Dropping incoming "
<< cricket::RtpPacketTypeToString(packet_type)
<< " packet: wrong size=" << len;
return;
}
rtc::CopyOnWriteBuffer packet(data, len);
if (packet_type == cricket::RtpPacketType::kRtcp) {
OnRtcpPacketReceived(std::move(packet), packet_time_us); //注意这里
} else {
OnRtpPacketReceived(std::move(packet), packet_time_us); // 多态 SrtpTransport::OnRtpPacketReceived
}
}
void SrtpTransport::OnRtpPacketReceived(rtc::CopyOnWriteBuffer packet,
int64_t packet_time_us) {
if (!IsSrtpActive()) {
RTC_LOG(LS_WARNING)
<< "Inactive SRTP transport received an RTP packet. Drop it.";
return;
}
TRACE_EVENT0("webrtc", "SRTP Decode");
char* data = packet.data<char>();
int len = rtc::checked_cast<int>(packet.size());
if (!UnprotectRtp(data, len, &len)) { //注意这里
int seq_num = -1;
uint32_t ssrc = 0;
cricket::GetRtpSeqNum(data, len, &seq_num);
cricket::GetRtpSsrc(data, len, &ssrc);
// Limit the error logging to avoid excessive logs when there are lots of
// bad packets.
const int kFailureLogThrottleCount = 100;
if (decryption_failure_count_ % kFailureLogThrottleCount == 0) {
RTC_LOG(LS_ERROR) << "Failed to unprotect RTP packet: size=" << len
<< ", seqnum=" << seq_num << ", SSRC=" << ssrc
<< ", previous failure count: "
<< decryption_failure_count_;
}
++decryption_failure_count_;
return;
}
packet.SetSize(len);
DemuxPacket(std::move(packet), packet_time_us); //注意这里
}
void RtpTransport::DemuxPacket(rtc::CopyOnWriteBuffer packet,
int64_t packet_time_us) {
webrtc::RtpPacketReceived parsed_packet(&header_extension_map_);
if (!parsed_packet.Parse(std::move(packet))) { //注意这里
RTC_LOG(LS_ERROR)
<< "Failed to parse the incoming RTP packet before demuxing. Drop it.";
return;
}
if (packet_time_us != -1) {
parsed_packet.set_arrival_time_ms((packet_time_us + 500) / 1000);
}
if (!rtp_demuxer_.OnRtpPacket(parsed_packet)) { //注意这里
RTC_LOG(LS_WARNING) << "Failed to demux RTP packet: "
<< RtpDemuxer::DescribePacket(parsed_packet);
}
}
bool RtpDemuxer::OnRtpPacket(const RtpPacketReceived& packet) {
RtpPacketSinkInterface* sink = ResolveSink(packet);
if (sink != nullptr) {
sink->OnRtpPacket(packet); // 参考链接： https://blog.csdn.net/zhengbin6072/article/details/108411342
return true;
}
return false;
}
void BaseChannel::OnRtpPacket(const webrtc::RtpPacketReceived& parsed_packet) {
// Take packet time from the |parsed_packet|.
// RtpPacketReceived.arrival_time_ms = (timestamp_us + 500) / 1000;
int64_t packet_time_us = -1;
if (parsed_packet.arrival_time_ms() > 0) {
packet_time_us = parsed_packet.arrival_time_ms() * 1000;
}
if (!has_received_packet_) {
has_received_packet_ = true;
signaling_thread()->Post(RTC_FROM_HERE, this, MSG_FIRSTPACKETRECEIVED);
}
if (!srtp_active() && srtp_required_) {
// Our session description indicates that SRTP is required, but we got a
// packet before our SRTP filter is active. This means either that
// a) we got SRTP packets before we received the SDES keys, in which case
// we can't decrypt it anyway, or
// b) we got SRTP packets before DTLS completed on both the RTP and RTCP
// transports, so we haven't yet extracted keys, even if DTLS did
// complete on the transport that the packets are being sent on. It's
// really good practice to wait for both RTP and RTCP to be good to go
// before sending media, to prevent weird failure modes, so it's fine
// for us to just eat packets here. This is all sidestepped if RTCP mux
// is used anyway.
RTC_LOG(LS_WARNING) << "Can't process incoming RTP packet when "
"SRTP is inactive and crypto is required";
return;
}
auto packet_buffer = parsed_packet.Buffer();
invoker_.AsyncInvoke<void>(
RTC_FROM_HERE, worker_thread_, [this, packet_buffer, packet_time_us] {
RTC_DCHECK(worker_thread_->IsCurrent());
media_channel_->OnPacketReceived(packet_buffer, packet_time_us); // 以视频为例
});
}
void WebRtcVideoChannel::OnPacketReceived(rtc::CopyOnWriteBuffer packet,
int64_t packet_time_us) {
RTC_DCHECK_RUN_ON(&thread_checker_);
const webrtc::PacketReceiver::DeliveryStatus delivery_result =
call_->Receiver()->DeliverPacket(webrtc::MediaType::VIDEO, packet,
packet_time_us);
switch (delivery_result) {
case webrtc::PacketReceiver::DELIVERY_OK:
return;
case webrtc::PacketReceiver::DELIVERY_PACKET_ERROR:
return;
case webrtc::PacketReceiver::DELIVERY_UNKNOWN_SSRC:
break;
}
uint32_t ssrc = 0;
if (!GetRtpSsrc(packet.cdata(), packet.size(), &ssrc)) {
return;
}
if (unknown_ssrc_packet_buffer_) {
unknown_ssrc_packet_buffer_->AddPacket(ssrc, packet_time_us, packet);
return;
}
if (discard_unknown_ssrc_packets_) {
return;
}
int payload_type = 0;
if (!GetRtpPayloadType(packet.cdata(), packet.size(), &payload_type)) {
return;
}
// See if this payload_type is registered as one that usually gets its own
// SSRC (RTX) or at least is safe to drop either way (FEC). If it is, and
// it wasn't handled above by DeliverPacket, that means we don't know what
// stream it associates with, and we shouldn't ever create an implicit channel
// for these.
for (auto& codec : recv_codecs_) {
if (payload_type == codec.rtx_payload_type ||
payload_type == codec.ulpfec.red_rtx_payload_type ||
payload_type == codec.ulpfec.ulpfec_payload_type) {
return;
}
}
if (payload_type == recv_flexfec_payload_type_) {
return;
}
switch (unsignalled_ssrc_handler_->OnUnsignalledSsrc(this, ssrc)) {
case UnsignalledSsrcHandler::kDropPacket:
return;
case UnsignalledSsrcHandler::kDeliverPacket:
break;
}
if (call_->Receiver()->DeliverPacket(webrtc::MediaType::VIDEO, packet,
packet_time_us) !=
webrtc::PacketReceiver::DELIVERY_OK) { //注意这里
RTC_LOG(LS_WARNING) << "Failed to deliver RTP packet on re-delivery.";
return;
}
}
PacketReceiver::DeliveryStatus Call::DeliverPacket(
MediaType media_type,
rtc::CopyOnWriteBuffer packet,
int64_t packet_time_us) {
RTC_DCHECK_RUN_ON(&configuration_sequence_checker_);
if (IsRtcp(packet.cdata(), packet.size()))
return DeliverRtcp(media_type, packet.cdata(), packet.size());
return DeliverRtp(media_type, std::move(packet), packet_time_us);//
}
PacketReceiver::DeliveryStatus Call::DeliverRtp(MediaType media_type,
rtc::CopyOnWriteBuffer packet,
int64_t packet_time_us) {
TRACE_EVENT0("webrtc", "Call::DeliverRtp");
RtpPacketReceived parsed_packet;
if (!parsed_packet.Parse(std::move(packet)))
return DELIVERY_PACKET_ERROR;
if (packet_time_us != -1) {
if (receive_time_calculator_) {
// Repair packet_time_us for clock resets by comparing a new read of
// the same clock (TimeUTCMicros) to a monotonic clock reading.
packet_time_us = receive_time_calculator_->ReconcileReceiveTimes(
packet_time_us, rtc::TimeUTCMicros(), clock_->TimeInMicroseconds());
}
parsed_packet.set_arrival_time_ms((packet_time_us + 500) / 1000);
} else {
parsed_packet.set_arrival_time_ms(clock_->TimeInMilliseconds());
}
// We might get RTP keep-alive packets in accordance with RFC6263 section 4.6.
// These are empty (zero length payload) RTP packets with an unsignaled
// payload type.
const bool is_keep_alive_packet = parsed_packet.payload_size() == 0;
RTC_DCHECK(media_type == MediaType::AUDIO || media_type == MediaType::VIDEO ||
is_keep_alive_packet);
ReadLockScoped read_lock(*receive_crit_);
auto it = receive_rtp_config_.find(parsed_packet.Ssrc());
if (it == receive_rtp_config_.end()) {
RTC_LOG(LS_ERROR) << "receive_rtp_config_ lookup failed for ssrc "
<< parsed_packet.Ssrc();
// Destruction of the receive stream, including deregistering from the
// RtpDemuxer, is not protected by the |receive_crit_| lock. But
// deregistering in the |receive_rtp_config_| map is protected by that lock.
// So by not passing the packet on to demuxing in this case, we prevent
// incoming packets to be passed on via the demuxer to a receive stream
// which is being torned down.
return DELIVERY_UNKNOWN_SSRC;
}
parsed_packet.IdentifyExtensions(it->second.extensions);
NotifyBweOfReceivedPacket(parsed_packet, media_type);
// RateCounters expect input parameter as int, save it as int,
// instead of converting each time it is passed to RateCounter::Add below.
int length = static_cast<int>(parsed_packet.size());
if (media_type == MediaType::AUDIO) {
if (audio_receiver_controller_.OnRtpPacket(parsed_packet)) { //注意这里
received_bytes_per_second_counter_.Add(length);
received_audio_bytes_per_second_counter_.Add(length);
event_log_->Log(
std::make_unique<RtcEventRtpPacketIncoming>(parsed_packet));
const int64_t arrival_time_ms = parsed_packet.arrival_time_ms();
if (!first_received_rtp_audio_ms_) {
first_received_rtp_audio_ms_.emplace(arrival_time_ms);
}
last_received_rtp_audio_ms_.emplace(arrival_time_ms);
return DELIVERY_OK;
}
} else if (media_type == MediaType::VIDEO) {
parsed_packet.set_payload_type_frequency(kVideoPayloadTypeFrequency);
if (video_receiver_controller_.OnRtpPacket(parsed_packet)) { //注意这里
received_bytes_per_second_counter_.Add(length);
received_video_bytes_per_second_counter_.Add(length);
event_log_->Log(
std::make_unique<RtcEventRtpPacketIncoming>(parsed_packet));
const int64_t arrival_time_ms = parsed_packet.arrival_time_ms();
if (!first_received_rtp_video_ms_) {
first_received_rtp_video_ms_.emplace(arrival_time_ms);
}
last_received_rtp_video_ms_.emplace(arrival_time_ms);
return DELIVERY_OK;
}
}
return DELIVERY_UNKNOWN_SSRC;
}
webrtc::VideoReceiveStream* Call::CreateVideoReceiveStream(
webrtc::VideoReceiveStream::Config configuration) {
TRACE_EVENT0("webrtc", "Call::CreateVideoReceiveStream");
RTC_DCHECK_RUN_ON(&configuration_sequence_checker_);
receive_side_cc_.SetSendPeriodicFeedback(
SendPeriodicFeedback(configuration.rtp.extensions));
RegisterRateObserver();
VideoReceiveStream* receive_stream = new VideoReceiveStream(
task_queue_factory_, &video_receiver_controller_, num_cpu_cores_, // 将 video_receiver_controller_ 传递进去
transport_send_ptr_->packet_router(), std::move(configuration),
module_process_thread_.get(), call_stats_.get(), clock_);
const webrtc::VideoReceiveStream::Config& config = receive_stream->config();
{
WriteLockScoped write_lock(*receive_crit_);
if (config.rtp.rtx_ssrc) {
// We record identical config for the rtx stream as for the main
// stream. Since the transport_send_cc negotiation is per payload
// type, we may get an incorrect value for the rtx stream, but
// that is unlikely to matter in practice.
receive_rtp_config_.emplace(config.rtp.rtx_ssrc,
ReceiveRtpConfig(config));
}
receive_rtp_config_.emplace(config.rtp.remote_ssrc,
ReceiveRtpConfig(config));
video_receive_streams_.insert(receive_stream);
ConfigureSync(config.sync_group);
}
receive_stream->SignalNetworkState(video_network_state_);
UpdateAggregateNetworkState();
event_log_->Log(std::make_unique<RtcEventVideoReceiveStreamConfig>(
CreateRtcLogStreamConfig(config)));
return receive_stream;
}
bool RtpStreamReceiverController::OnRtpPacket(const RtpPacketReceived& packet) {
rtc::CritScope cs(&lock_);
return demuxer_.OnRtpPacket(packet); //注意这里
}
bool RtpDemuxer::OnRtpPacket(const RtpPacketReceived& packet) {
RtpPacketSinkInterface* sink = ResolveSink(packet);
if (sink != nullptr) {
sink->OnRtpPacket(packet); // 这里的 sink 实际上就是 webrtc::RtpVideoStreamReceiver this 指针
return true;
}
return false;
}
VideoReceiveStream::VideoReceiveStream
===>
media_receiver_ = receiver_controller->CreateReceiver(
config_.rtp.remote_ssrc, &rtp_video_stream_receiver_); // rtp_video_stream_receiver_ 就是 webrtc::RtpVideoStreamReceiver
std::unique_ptr<RtpStreamReceiverInterface>
RtpStreamReceiverController::CreateReceiver(uint32_t ssrc,
RtpPacketSinkInterface* sink) {
return std::make_unique<Receiver>(this, ssrc, sink); //
}
RtpStreamReceiverController::Receiver::Receiver(
RtpStreamReceiverController* controller,
uint32_t ssrc,
RtpPacketSinkInterface* sink)
: controller_(controller), sink_(sink) {
const bool sink_added = controller_->AddSink(ssrc, sink_); // sink 就是 webrtc::RtpVideoStreamReceiver
if (!sink_added) {
RTC_LOG(LS_ERROR)
<< "RtpStreamReceiverController::Receiver::Receiver: Sink "
<< "could not be added for SSRC=" << ssrc << ".";
}
}
void RtpVideoStreamReceiver::OnRtpPacket(const RtpPacketReceived& packet) {
RTC_DCHECK_RUN_ON(&worker_task_checker_);
if (!receiving_) {
return;
}
if (!packet.recovered()) {
// TODO(nisse): Exclude out-of-order packets?
int64_t now_ms = clock_->TimeInMilliseconds();
{
rtc::CritScope cs(&sync_info_lock_);
last_received_rtp_timestamp_ = packet.Timestamp();
last_received_rtp_system_time_ms_ = now_ms;
}
// Periodically log the RTP header of incoming packets.
if (now_ms - last_packet_log_ms_ > kPacketLogIntervalMs) {
rtc::StringBuilder ss;
ss << "Packet received on SSRC: " << packet.Ssrc()
<< " with payload type: " << static_cast<int>(packet.PayloadType())
<< ", timestamp: " << packet.Timestamp()
<< ", sequence number: " << packet.SequenceNumber()
<< ", arrival time: " << packet.arrival_time_ms();
int32_t time_offset;
if (packet.GetExtension<TransmissionOffset>(&time_offset)) {
ss << ", toffset: " << time_offset;
}
uint32_t send_time;
if (packet.GetExtension<AbsoluteSendTime>(&send_time)) {
ss << ", abs send time: " << send_time;
}
RTC_LOG(LS_INFO) << ss.str();
last_packet_log_ms_ = now_ms;
}
}
ReceivePacket(packet); //注意这里
// Update receive statistics after ReceivePacket.
// Receive statistics will be reset if the payload type changes (make sure
// that the first packet is included in the stats).
if (!packet.recovered()) {
rtp_receive_statistics_->OnRtpPacket(packet);
}
for (RtpPacketSinkInterface* secondary_sink : secondary_sinks_) {
secondary_sink->OnRtpPacket(packet); //注意这里
}
}
void RtpVideoStreamReceiver::ReceivePacket(const RtpPacketReceived& packet) {
if (packet.payload_size() == 0) {
// Padding or keep-alive packet.
// TODO(nisse): Could drop empty packets earlier, but need to figure out how
// they should be counted in stats.
NotifyReceiverOfEmptyPacket(packet.SequenceNumber());
return;
}
if (packet.PayloadType() == config_.rtp.red_payload_type) {
ParseAndHandleEncapsulatingHeader(packet); // FEC 以及数据恢复相关
return;
}
const auto type_it = payload_type_map_.find(packet.PayloadType());
if (type_it == payload_type_map_.end()) {
return;
}
auto depacketizer =
absl::WrapUnique(RtpDepacketizer::Create(type_it->second)); // 创建对应视频编码格式的解包器
if (!depacketizer) {
RTC_LOG(LS_ERROR) << "Failed to create depacketizer.";
return;
}
RtpDepacketizer::ParsedPayload parsed_payload;
if (!depacketizer->Parse(&parsed_payload, packet.payload().data(), // RtpDepacketizerH264::Parse 或者 RtpDepacketizerVp8::Parse
packet.payload().size())) {
RTC_LOG(LS_WARNING) << "Failed parsing payload.";
return;
}
RTPHeader rtp_header;
packet.GetHeader(&rtp_header);
RTPVideoHeader video_header = parsed_payload.video_header();
video_header.rotation = kVideoRotation_0;
video_header.content_type = VideoContentType::UNSPECIFIED;
video_header.video_timing.flags = VideoSendTiming::kInvalid;
video_header.is_last_packet_in_frame = rtp_header.markerBit;
video_header.frame_marking.temporal_id = kNoTemporalIdx;
if (parsed_payload.video_header().codec == kVideoCodecVP9) {
const RTPVideoHeaderVP9& codec_header = absl::get<RTPVideoHeaderVP9>(
parsed_payload.video_header().video_type_header);
video_header.is_last_packet_in_frame |= codec_header.end_of_frame;
video_header.is_first_packet_in_frame |= codec_header.beginning_of_frame;
}
packet.GetExtension<VideoOrientation>(&video_header.rotation);
packet.GetExtension<VideoContentTypeExtension>(&video_header.content_type);
packet.GetExtension<VideoTimingExtension>(&video_header.video_timing);
packet.GetExtension<PlayoutDelayLimits>(&video_header.playout_delay);
packet.GetExtension<FrameMarkingExtension>(&video_header.frame_marking);
// Color space should only be transmitted in the last packet of a frame,
// therefore, neglect it otherwise so that last_color_space_ is not reset by
// mistake.
if (video_header.is_last_packet_in_frame) {
video_header.color_space = packet.GetExtension<ColorSpaceExtension>();
if (video_header.color_space ||
video_header.frame_type == VideoFrameType::kVideoFrameKey) {
// Store color space since it's only transmitted when changed or for key
// frames. Color space will be cleared if a key frame is transmitted
// without color space information.
last_color_space_ = video_header.color_space;
} else if (last_color_space_) {
video_header.color_space = last_color_space_;
}
}
absl::optional<RtpGenericFrameDescriptor> generic_descriptor_wire;
generic_descriptor_wire.emplace();
const bool generic_descriptor_v00 =
packet.GetExtension<RtpGenericFrameDescriptorExtension00>(
&generic_descriptor_wire.value());
const bool generic_descriptor_v01 =
packet.GetExtension<RtpGenericFrameDescriptorExtension01>(
&generic_descriptor_wire.value());
if (generic_descriptor_v00 && generic_descriptor_v01) {
RTC_LOG(LS_WARNING) << "RTP packet had two different GFD versions.";
return;
}
if (generic_descriptor_v00 || generic_descriptor_v01) {
if (generic_descriptor_v00) {
generic_descriptor_wire->SetByteRepresentation(
packet.GetRawExtension<RtpGenericFrameDescriptorExtension00>());
} else {
generic_descriptor_wire->SetByteRepresentation(
packet.GetRawExtension<RtpGenericFrameDescriptorExtension01>());
}
video_header.is_first_packet_in_frame =
generic_descriptor_wire->FirstPacketInSubFrame();
video_header.is_last_packet_in_frame =
rtp_header.markerBit || generic_descriptor_wire->LastPacketInSubFrame();
if (generic_descriptor_wire->FirstPacketInSubFrame()) {
video_header.frame_type =
generic_descriptor_wire->FrameDependenciesDiffs().empty()
? VideoFrameType::kVideoFrameKey
: VideoFrameType::kVideoFrameDelta;
}
video_header.width = generic_descriptor_wire->Width();
video_header.height = generic_descriptor_wire->Height();
} else {
generic_descriptor_wire.reset();
}
OnReceivedPayloadData(parsed_payload.payload, parsed_payload.payload_length, //注意这里
rtp_header, video_header, generic_descriptor_wire,
packet.recovered());
}
int32_t RtpVideoStreamReceiver::OnReceivedPayloadData(
const uint8_t* payload_data,
size_t payload_size,
const RTPHeader& rtp_header,
const RTPVideoHeader& video_header,
const absl::optional<RtpGenericFrameDescriptor>& generic_descriptor,
bool is_recovered) {
VCMPacket packet(payload_data, payload_size, rtp_header, video_header,
ntp_estimator_.Estimate(rtp_header.timestamp), //RemoteNtpTimeEstimator::Estimate 将 RTP 时间转换成 NTP 时间
clock_->TimeInMilliseconds());
packet.generic_descriptor = generic_descriptor;
if (loss_notification_controller_) {
if (is_recovered) {
// TODO(bugs.webrtc.org/10336): Implement support for reordering.
RTC_LOG(LS_INFO)
<< "LossNotificationController does not support reordering.";
} else if (!generic_descriptor) {
RTC_LOG(LS_WARNING) << "LossNotificationController requires generic "
"frame descriptor, but it is missing.";
} else {
loss_notification_controller_->OnReceivedPacket(rtp_header.sequenceNumber,
*generic_descriptor);
}
}
if (nack_module_) {
const bool is_keyframe =
video_header.is_first_packet_in_frame &&
video_header.frame_type == VideoFrameType::kVideoFrameKey;
packet.timesNacked = nack_module_->OnReceivedPacket(
rtp_header.sequenceNumber, is_keyframe, is_recovered);
} else {
packet.timesNacked = -1;
}
if (packet.sizeBytes == 0) {
NotifyReceiverOfEmptyPacket(packet.seqNum);
rtcp_feedback_buffer_.SendBufferedRtcpFeedback();
return 0;
}
if (packet.codec() == kVideoCodecH264) {
// Only when we start to receive packets will we know what payload type
// that will be used. When we know the payload type insert the correct
// sps/pps into the tracker.
if (packet.payloadType != last_payload_type_) {
last_payload_type_ = packet.payloadType;
InsertSpsPpsIntoTracker(packet.payloadType);
}
switch (tracker_.CopyAndFixBitstream(&packet)) { //CopyAndFixBitstream 这里是关键,从新分配了buffer保存在了VCMPacket里面,并插入了start code
case video_coding::H264SpsPpsTracker::kRequestKeyframe:
rtcp_feedback_buffer_.RequestKeyFrame();
rtcp_feedback_buffer_.SendBufferedRtcpFeedback();
RTC_FALLTHROUGH();
case video_coding::H264SpsPpsTracker::kDrop:
return 0;
case video_coding::H264SpsPpsTracker::kInsert:
break;
}
} else {
uint8_t* data = new uint8_t[packet.sizeBytes];
memcpy(data, packet.dataPtr, packet.sizeBytes);
packet.dataPtr = data;
}
rtcp_feedback_buffer_.SendBufferedRtcpFeedback();
if (!packet_buffer_.InsertPacket(&packet)) { //video_coding::PacketBuffer packet_buffer_;
RequestKeyFrame();
}
return 0;
}
bool PacketBuffer::InsertPacket(VCMPacket* packet) {
std::vector<std::unique_ptr<RtpFrameObject>> found_frames;
{
rtc::CritScope lock(&crit_);
OnTimestampReceived(packet->timestamp);
uint16_t seq_num = packet->seqNum;
size_t index = seq_num % size_;
if (!first_packet_received_) {
first_seq_num_ = seq_num;
first_packet_received_ = true;
} else if (AheadOf(first_seq_num_, seq_num)) {
// If we have explicitly cleared past this packet then it's old,
// don't insert it, just silently ignore it.
if (is_cleared_to_first_seq_num_) {
delete[] packet->dataPtr;
packet->dataPtr = nullptr;
return true;
}
first_seq_num_ = seq_num;
}
if (sequence_buffer_[index].used) {
// Duplicate packet, just delete the payload.
if (data_buffer_[index].seqNum == packet->seqNum) {
delete[] packet->dataPtr;
packet->dataPtr = nullptr;
return true;
}
// The packet buffer is full, try to expand the buffer.
while (ExpandBufferSize() && sequence_buffer_[seq_num % size_].used) {
}
index = seq_num % size_;
// Packet buffer is still full since we were unable to expand the buffer.
if (sequence_buffer_[index].used) {
// Clear the buffer, delete payload, and return false to signal that a
// new keyframe is needed.
RTC_LOG(LS_WARNING) << "Clear PacketBuffer and request key frame.";
Clear();
delete[] packet->dataPtr;
packet->dataPtr = nullptr;
return false;
}
}
sequence_buffer_[index].frame_begin = packet->is_first_packet_in_frame();
sequence_buffer_[index].frame_end = packet->is_last_packet_in_frame();
sequence_buffer_[index].seq_num = packet->seqNum;
sequence_buffer_[index].continuous = false;
sequence_buffer_[index].frame_created = false;
sequence_buffer_[index].used = true;
data_buffer_[index] = *packet;
packet->dataPtr = nullptr; // 转移了指针的所有者
UpdateMissingPackets(packet->seqNum); //注意这里
int64_t now_ms = clock_->TimeInMilliseconds();
last_received_packet_ms_ = now_ms;
if (packet->video_header.frame_type == VideoFrameType::kVideoFrameKey)
last_received_keyframe_packet_ms_ = now_ms;
found_frames = FindFrames(seq_num); // 参考链接： https://blog.csdn.net/sonysuqin/article/details/106629343
}
for (std::unique_ptr<RtpFrameObject>& frame : found_frames)
assembled_frame_callback_->OnAssembledFrame(std::move(frame)); //注意这里
return true;
}
std::vector<std::unique_ptr<RtpFrameObject>> PacketBuffer::FindFrames(
uint16_t seq_num) {
std::vector<std::unique_ptr<RtpFrameObject>> found_frames;
for (size_t i = 0; i < size_ && PotentialNewFrame(seq_num); ++i) { //注意这里
size_t index = seq_num % size_;
sequence_buffer_[index].continuous = true;
// If all packets of the frame is continuous, find the first packet of the
// frame and create an RtpFrameObject.
if (sequence_buffer_[index].frame_end) {
size_t frame_size = 0;
int max_nack_count = -1;
uint16_t start_seq_num = seq_num;
int64_t min_recv_time = data_buffer_[index].packet_info.receive_time_ms();
int64_t max_recv_time = data_buffer_[index].packet_info.receive_time_ms();
RtpPacketInfos::vector_type packet_infos;
// Find the start index by searching backward until the packet with
// the |frame_begin| flag is set.
int start_index = index;
size_t tested_packets = 0;
int64_t frame_timestamp = data_buffer_[start_index].timestamp;
// Identify H.264 keyframes by means of SPS, PPS, and IDR.
bool is_h264 = data_buffer_[start_index].codec() == kVideoCodecH264;
bool has_h264_sps = false;
bool has_h264_pps = false;
bool has_h264_idr = false;
bool is_h264_keyframe = false;
while (true) {
++tested_packets;
frame_size += data_buffer_[start_index].sizeBytes;
max_nack_count =
std::max(max_nack_count, data_buffer_[start_index].timesNacked);
sequence_buffer_[start_index].frame_created = true;
min_recv_time =
std::min(min_recv_time,
data_buffer_[start_index].packet_info.receive_time_ms());
max_recv_time =
std::max(max_recv_time,
data_buffer_[start_index].packet_info.receive_time_ms());
// Should use |push_front()| since the loop traverses backwards. But
// it's too inefficient to do so on a vector so we'll instead fix the
// order afterwards.
packet_infos.push_back(data_buffer_[start_index].packet_info);
if (!is_h264 && sequence_buffer_[start_index].frame_begin)
break;
if (is_h264) {
const auto* h264_header = absl::get_if<RTPVideoHeaderH264>(
&data_buffer_[start_index].video_header.video_type_header);
if (!h264_header || h264_header->nalus_length >= kMaxNalusPerPacket)
return found_frames;
for (size_t j = 0; j < h264_header->nalus_length; ++j) {
if (h264_header->nalus[j].type == H264::NaluType::kSps) {
has_h264_sps = true;
} else if (h264_header->nalus[j].type == H264::NaluType::kPps) {
has_h264_pps = true;
} else if (h264_header->nalus[j].type == H264::NaluType::kIdr) {
has_h264_idr = true;
}
}
if ((sps_pps_idr_is_h264_keyframe_ && has_h264_idr && has_h264_sps &&
has_h264_pps) ||
(!sps_pps_idr_is_h264_keyframe_ && has_h264_idr)) {
is_h264_keyframe = true;
}
}
if (tested_packets == size_)
break;
start_index = start_index > 0 ? start_index - 1 : size_ - 1;
// In the case of H264 we don't have a frame_begin bit (yes,
// |frame_begin| might be set to true but that is a lie). So instead
// we traverese backwards as long as we have a previous packet and
// the timestamp of that packet is the same as this one. This may cause
// the PacketBuffer to hand out incomplete frames.
// See: https://bugs.chromium.org/p/webrtc/issues/detail?id=7106
if (is_h264 &&
(!sequence_buffer_[start_index].used ||
data_buffer_[start_index].timestamp != frame_timestamp)) {
break;
}
--start_seq_num;
}
// Fix the order since the packet-finding loop traverses backwards.
std::reverse(packet_infos.begin(), packet_infos.end());
if (is_h264) {
// Warn if this is an unsafe frame.
if (has_h264_idr && (!has_h264_sps || !has_h264_pps)) {
RTC_LOG(LS_WARNING)
<< "Received H.264-IDR frame "
<< "(SPS: " << has_h264_sps << ", PPS: " << has_h264_pps
<< "). Treating as "
<< (sps_pps_idr_is_h264_keyframe_ ? "delta" : "key")
<< " frame since WebRTC-SpsPpsIdrIsH264Keyframe is "
<< (sps_pps_idr_is_h264_keyframe_ ? "enabled." : "disabled");
}
// Now that we have decided whether to treat this frame as a key frame
// or delta frame in the frame buffer, we update the field that
// determines if the RtpFrameObject is a key frame or delta frame.
const size_t first_packet_index = start_seq_num % size_;
RTC_CHECK_LT(first_packet_index, size_);
if (is_h264_keyframe) {
data_buffer_[first_packet_index].video_header.frame_type =
VideoFrameType::kVideoFrameKey;
} else {
data_buffer_[first_packet_index].video_header.frame_type =
VideoFrameType::kVideoFrameDelta;
}
// With IPPP, if this is not a keyframe, make sure there are no gaps
// in the packet sequence numbers up until this point.
const uint8_t h264tid =
data_buffer_[start_index].video_header.frame_marking.temporal_id;
if (h264tid == kNoTemporalIdx && !is_h264_keyframe &&
missing_packets_.upper_bound(start_seq_num) !=
missing_packets_.begin()) {
uint16_t stop_index = (index + 1) % size_;
while (start_index != stop_index) {
sequence_buffer_[start_index].frame_created = false;
start_index = (start_index + 1) % size_;
}
return found_frames;
}
}
missing_packets_.erase(missing_packets_.begin(),
missing_packets_.upper_bound(seq_num));
const VCMPacket* first_packet = GetPacket(start_seq_num);
const VCMPacket* last_packet = GetPacket(seq_num);
auto frame = std::make_unique<RtpFrameObject>( // 创建 RtpFrameObject
start_seq_num, seq_num, last_packet->markerBit, max_nack_count,
min_recv_time, max_recv_time, first_packet->timestamp,
first_packet->ntp_time_ms_, last_packet->video_header.video_timing,
first_packet->payloadType, first_packet->codec(),
last_packet->video_header.rotation,
last_packet->video_header.content_type, first_packet->video_header,
last_packet->video_header.color_space,
first_packet->generic_descriptor,
RtpPacketInfos(std::move(packet_infos)),
GetEncodedImageBuffer(frame_size, start_seq_num, seq_num)); //GetEncodedImageBuffer 将 PacketBuffer 中的具体的媒体数据 data_buffer_ 拷贝到新创建的 EncodedImageBuffer 中
// RtpFrameObject(实际上是父类webrtc::EncodedImage) 含有 tc::scoped_refptr<EncodedImageBufferInterface> encoded_data_,用来保存
found_frames.emplace_back(std::move(frame)); // GetEncodedImageBuffer 返回的 EncodedImageBuffer
ClearInterval(start_seq_num, seq_num); // 由于上面的 GetEncodedImageBuffer 已经将指定 seq 段的 VCMPacket 中的媒体数据(data_buffer_[index].dataPtr)拷贝到了相应的 EncodedImageBuffer 中，所以这里将这些媒体数据进行释放掉。
// PacketBuffer 中的 std::vector<VCMPacket> data_buffer_ 缓存媒体数据，而 VCMPacket 中的 const uint8_t* dataPtr 保存最终的媒体数据。
} // 由此可见 webrtc::video_coding::RtpFrameObject 实际上保存的是一个完整帧的 buffer，这个buffer由多个 VCMPacket 中的媒体数据组成。
++seq_num;
}
return found_frames;
}
//webrtc::video_coding::RtpFrameObject 的继承关系
class webrtc::VCMEncodedFrame : protected webrtc::EncodedImage
class webrtc::video_coding::EncodedFrame : public webrtc::VCMEncodedFrame
class webrtc::video_coding::RtpFrameObject : public webrtc::video_coding::EncodedFrame
// class webrtc::EncodedImageBuffer : public webrtc::EncodedImageBufferInterface
rtc::scoped_refptr<EncodedImageBuffer> PacketBuffer::GetEncodedImageBuffer(
size_t frame_size,
uint16_t first_seq_num,
uint16_t last_seq_num) {
size_t index = first_seq_num % size_;
size_t end = (last_seq_num + 1) % size_;
auto buffer = EncodedImageBuffer::Create(frame_size); //
size_t offset = 0;
do {
RTC_DCHECK(sequence_buffer_[index].used);
size_t length = data_buffer_[index].sizeBytes;
RTC_CHECK_LE(offset + length, buffer->size());
memcpy(buffer->data() + offset, data_buffer_[index].dataPtr, length);
offset += length;
index = (index + 1) % size_;
} while (index != end);
return buffer; //
}
rtc::scoped_refptr<EncodedImageBuffer> EncodedImageBuffer::Create(size_t size) {
return new rtc::RefCountedObject<EncodedImageBuffer>(size); //
}
class EncodedImageBuffer : public EncodedImageBufferInterface {
public:
static rtc::scoped_refptr<EncodedImageBuffer> Create() { return Create(0); }
static rtc::scoped_refptr<EncodedImageBuffer> Create(size_t size);
static rtc::scoped_refptr<EncodedImageBuffer> Create(const uint8_t* data,
size_t size);
const uint8_t* data() const override;
uint8_t* data() override;
size_t size() const override;
void Realloc(size_t t) override;
protected:
explicit EncodedImageBuffer(size_t size);
EncodedImageBuffer(const uint8_t* data, size_t size);
~EncodedImageBuffer();
size_t size_; //
uint8_t* buffer_; //
};
RtpFrameObject::RtpFrameObject(
uint16_t first_seq_num,
uint16_t last_seq_num,
bool markerBit,
int times_nacked,
int64_t first_packet_received_time,
int64_t last_packet_received_time,
uint32_t rtp_timestamp,
int64_t ntp_time_ms,
const VideoSendTiming& timing,
uint8_t payload_type,
VideoCodecType codec,
VideoRotation rotation,
VideoContentType content_type,
const RTPVideoHeader& video_header,
const absl::optional<webrtc::ColorSpace>& color_space,
const absl::optional<RtpGenericFrameDescriptor>& generic_descriptor,
RtpPacketInfos packet_infos,
rtc::scoped_refptr<EncodedImageBuffer> image_buffer) //注意这里
: first_seq_num_(first_seq_num),
last_seq_num_(last_seq_num),
last_packet_received_time_(last_packet_received_time),
times_nacked_(times_nacked) {
rtp_video_header_ = video_header;
rtp_generic_frame_descriptor_ = generic_descriptor;
// EncodedFrame members
codec_type_ = codec;
// TODO(philipel): Remove when encoded image is replaced by EncodedFrame.
// VCMEncodedFrame members
CopyCodecSpecific(&rtp_video_header_);
_completeFrame = true;
_payloadType = payload_type;
SetTimestamp(rtp_timestamp);
ntp_time_ms_ = ntp_time_ms;
_frameType = rtp_video_header_.frame_type;
// Setting frame's playout delays to the same values
// as of the first packet's.
SetPlayoutDelay(rtp_video_header_.playout_delay);
SetEncodedData(std::move(image_buffer)); // SetEncodedData 实际上就是其父类 webrtc::EncodedImage::SetEncodedData
_encodedWidth = rtp_video_header_.width;
_encodedHeight = rtp_video_header_.height;
// EncodedFrame members
SetPacketInfos(std::move(packet_infos));
rotation_ = rotation;
SetColorSpace(color_space);
content_type_ = content_type;
if (timing.flags != VideoSendTiming::kInvalid) {
// ntp_time_ms_ may be -1 if not estimated yet. This is not a problem,
// as this will be dealt with at the time of reporting.
timing_.encode_start_ms = ntp_time_ms_ + timing.encode_start_delta_ms;
timing_.encode_finish_ms = ntp_time_ms_ + timing.encode_finish_delta_ms;
timing_.packetization_finish_ms =
ntp_time_ms_ + timing.packetization_finish_delta_ms;
timing_.pacer_exit_ms = ntp_time_ms_ + timing.pacer_exit_delta_ms;
timing_.network_timestamp_ms =
ntp_time_ms_ + timing.network_timestamp_delta_ms;
timing_.network2_timestamp_ms =
ntp_time_ms_ + timing.network2_timestamp_delta_ms;
}
timing_.receive_start_ms = first_packet_received_time;
timing_.receive_finish_ms = last_packet_received_time;
timing_.flags = timing.flags;
is_last_spatial_layer = markerBit;
}
// webrtc::EncodedImage::SetEncodedData
void SetEncodedData(
rtc::scoped_refptr<EncodedImageBufferInterface> encoded_data) {
encoded_data_ = encoded_data; //rtc::scoped_refptr<EncodedImageBufferInterface> encoded_data_;
size_ = encoded_data->size(); // encoded_data_ 是 webrtc::EncodedImage 的成员变量
buffer_ = nullptr;
}
void RtpVideoStreamReceiver::OnAssembledFrame(
std::unique_ptr<video_coding::RtpFrameObject> frame) {
RTC_DCHECK_RUN_ON(&network_tc_);
RTC_DCHECK(frame);
absl::optional<RtpGenericFrameDescriptor> descriptor =
frame->GetGenericFrameDescriptor();
if (loss_notification_controller_ && descriptor) {
loss_notification_controller_->OnAssembledFrame(
frame->first_seq_num(), descriptor->FrameId(),
descriptor->Discardable().value_or(false),
descriptor->FrameDependenciesDiffs());
}
// If frames arrive before a key frame, they would not be decodable.
// In that case, request a key frame ASAP.
if (!has_received_frame_) {
if (frame->FrameType() != VideoFrameType::kVideoFrameKey) {
// |loss_notification_controller_|, if present, would have already
// requested a key frame when the first packet for the non-key frame
// had arrived, so no need to replicate the request.
if (!loss_notification_controller_) {
RequestKeyFrame();
}
}
has_received_frame_ = true;
}
rtc::CritScope lock(&reference_finder_lock_);
// Reset |reference_finder_| if |frame| is new and the codec have changed.
if (current_codec_) {
bool frame_is_newer =
AheadOf(frame->Timestamp(), last_assembled_frame_rtp_timestamp_);
if (frame->codec_type() != current_codec_) {
if (frame_is_newer) {
// When we reset the |reference_finder_| we don't want new picture ids
// to overlap with old picture ids. To ensure that doesn't happen we
// start from the |last_completed_picture_id_| and add an offset in case
// of reordering.
reference_finder_ =
std::make_unique<video_coding::RtpFrameReferenceFinder>(
this, last_completed_picture_id_ +
std::numeric_limits<uint16_t>::max());
current_codec_ = frame->codec_type();
} else {
// Old frame from before the codec switch, discard it.
return;
}
}
if (frame_is_newer) {
last_assembled_frame_rtp_timestamp_ = frame->Timestamp();
}
} else {
current_codec_ = frame->codec_type();
last_assembled_frame_rtp_timestamp_ = frame->Timestamp();
}
if (buffered_frame_decryptor_ == nullptr) {
reference_finder_->ManageFrame(std::move(frame)); // RtpFrameReferenceFinder::ManageFrame
} else {
buffered_frame_decryptor_->ManageEncryptedFrame(std::move(frame));
}
}
void RtpFrameReferenceFinder::ManageFrame(
std::unique_ptr<RtpFrameObject> frame) {
// If we have cleared past this frame, drop it.
if (cleared_to_seq_num_ != -1 &&
AheadOf<uint16_t>(cleared_to_seq_num_, frame->first_seq_num())) {
return;
}
FrameDecision decision = ManageFrameInternal(frame.get()); //注意这里
switch (decision) {
case kStash:
if (stashed_frames_.size() > kMaxStashedFrames)
stashed_frames_.pop_back();
stashed_frames_.push_front(std::move(frame));
break;
case kHandOff:
HandOffFrame(std::move(frame)); //注意这里
RetryStashedFrames();
break;
case kDrop:
break;
}
}
RtpFrameReferenceFinder::FrameDecision
RtpFrameReferenceFinder::ManageFrameInternal(RtpFrameObject* frame) {
absl::optional<RtpGenericFrameDescriptor> generic_descriptor =
frame->GetGenericFrameDescriptor();
if (generic_descriptor) {
return ManageFrameGeneric(frame, *generic_descriptor);
}
switch (frame->codec_type()) {
case kVideoCodecVP8:
return ManageFrameVp8(frame); //注意这里
case kVideoCodecVP9:
return ManageFrameVp9(frame); //注意这里
case kVideoCodecH264:
return ManageFrameH264(frame); //注意这里
default: {
// Use 15 first bits of frame ID as picture ID if available.
const RTPVideoHeader& video_header = frame->GetRtpVideoHeader();
int picture_id = kNoPictureId;
if (video_header.generic)
picture_id = video_header.generic->frame_id & 0x7fff;
return ManageFramePidOrSeqNum(frame, picture_id); //注意这里
}
}
}
RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameH264(
RtpFrameObject* frame) {
const FrameMarking& rtp_frame_marking = frame->GetFrameMarking();
uint8_t tid = rtp_frame_marking.temporal_id;
bool blSync = rtp_frame_marking.base_layer_sync;
if (tid == kNoTemporalIdx)
return ManageFramePidOrSeqNum(std::move(frame), kNoPictureId); //注意这里
frame->id.picture_id = frame->last_seq_num();
if (frame->frame_type() == VideoFrameType::kVideoFrameKey) {
// For H264, use last_seq_num_gop_ to simply store last picture id
// as a pair of unpadded and padded sequence numbers.
if (last_seq_num_gop_.empty()) {
last_seq_num_gop_.insert(std::make_pair(
0, std::make_pair(frame->id.picture_id, frame->id.picture_id)));
}
}
// Stash if we have no keyframe yet.
if (last_seq_num_gop_.empty())
return kStash;
// Check for gap in sequence numbers. Store in |not_yet_received_seq_num_|.
if (frame->frame_type() == VideoFrameType::kVideoFrameDelta) {
uint16_t last_pic_id_padded = last_seq_num_gop_.begin()->second.second;
if (AheadOf<uint16_t>(frame->id.picture_id, last_pic_id_padded)) {
do {
last_pic_id_padded = last_pic_id_padded + 1;
not_yet_received_seq_num_.insert(last_pic_id_padded);
} while (last_pic_id_padded != frame->id.picture_id);
}
}
int64_t unwrapped_tl0 = tl0_unwrapper_.Unwrap(rtp_frame_marking.tl0_pic_idx);
// Clean up info for base layers that are too old.
int64_t old_tl0_pic_idx = unwrapped_tl0 - kMaxLayerInfo;
auto clean_layer_info_to = layer_info_.lower_bound(old_tl0_pic_idx);
layer_info_.erase(layer_info_.begin(), clean_layer_info_to);
// Clean up info about not yet received frames that are too old.
uint16_t old_picture_id = frame->id.picture_id - kMaxNotYetReceivedFrames * 2;
auto clean_frames_to = not_yet_received_seq_num_.lower_bound(old_picture_id);
not_yet_received_seq_num_.erase(not_yet_received_seq_num_.begin(),
clean_frames_to);
if (frame->frame_type() == VideoFrameType::kVideoFrameKey) {
frame->num_references = 0;
layer_info_[unwrapped_tl0].fill(-1);
UpdateDataH264(frame, unwrapped_tl0, tid);
return kHandOff;
}
auto layer_info_it =
layer_info_.find(tid == 0 ? unwrapped_tl0 - 1 : unwrapped_tl0);
// Stash if we have no base layer frame yet.
if (layer_info_it == layer_info_.end())
return kStash;
// Base layer frame. Copy layer info from previous base layer frame.
if (tid == 0) {
layer_info_it =
layer_info_.insert(std::make_pair(unwrapped_tl0, layer_info_it->second))
.first;
frame->num_references = 1;
frame->references[0] = layer_info_it->second[0];
UpdateDataH264(frame, unwrapped_tl0, tid);
return kHandOff;
}
// This frame only references its base layer frame.
if (blSync) {
frame->num_references = 1;
frame->references[0] = layer_info_it->second[0];
UpdateDataH264(frame, unwrapped_tl0, tid);
return kHandOff;
}
// Find all references for general frame.
frame->num_references = 0;
for (uint8_t layer = 0; layer <= tid; ++layer) {
// Stash if we have not yet received frames on this temporal layer.
if (layer_info_it->second[layer] == -1)
return kStash;
// Drop if the last frame on this layer is ahead of this frame. A layer sync
// frame was received after this frame for the same base layer frame.
uint16_t last_frame_in_layer = layer_info_it->second[layer];
if (AheadOf<uint16_t>(last_frame_in_layer, frame->id.picture_id))
return kDrop;
// Stash and wait for missing frame between this frame and the reference
auto not_received_seq_num_it =
not_yet_received_seq_num_.upper_bound(last_frame_in_layer);
if (not_received_seq_num_it != not_yet_received_seq_num_.end() &&
AheadOf<uint16_t>(frame->id.picture_id, *not_received_seq_num_it)) {
return kStash;
}
if (!(AheadOf<uint16_t>(frame->id.picture_id, last_frame_in_layer))) {
RTC_LOG(LS_WARNING) << "Frame with picture id " << frame->id.picture_id
<< " and packet range [" << frame->first_seq_num()
<< ", " << frame->last_seq_num()
<< "] already received, "
<< " dropping frame.";
return kDrop;
}
++frame->num_references;
frame->references[layer] = last_frame_in_layer;
}
UpdateDataH264(frame, unwrapped_tl0, tid);
return kHandOff;
}
void RtpFrameReferenceFinder::HandOffFrame(
std::unique_ptr<RtpFrameObject> frame) {
frame->id.picture_id += picture_id_offset_;
for (size_t i = 0; i < frame->num_references; ++i) {
frame->references[i] += picture_id_offset_;
}
frame_callback_->OnCompleteFrame(std::move(frame)); //注意这里
}
// webrtc::video_coding::EncodedFrame 的继承关系
class webrtc::VCMEncodedFrame : protected webrtc::EncodedImage
class webrtc::video_coding::EncodedFrame : public webrtc::VCMEncodedFrame
class webrtc::video_coding::RtpFrameObject : public webrtc::video_coding::EncodedFrame
void RtpVideoStreamReceiver::OnCompleteFrame(
std::unique_ptr<video_coding::EncodedFrame> frame) {
{
rtc::CritScope lock(&last_seq_num_cs_);
video_coding::RtpFrameObject* rtp_frame =
static_cast<video_coding::RtpFrameObject*>(frame.get());
last_seq_num_for_pic_id_[rtp_frame->id.picture_id] =
rtp_frame->last_seq_num();
}
last_completed_picture_id_ =
std::max(last_completed_picture_id_, frame->id.picture_id);
complete_frame_callback_->OnCompleteFrame(std::move(frame)); //complete_frame_callback_ 是在 RtpVideoStreamReceiver 的构造函数中传递进来的，具体参考 VideoReceiveStream 的构造函数
} // complete_frame_callback_ 实际上就是 VideoReceiveStream 的 this 指针
void VideoReceiveStream::OnCompleteFrame(
std::unique_ptr<video_coding::EncodedFrame> frame) {
RTC_DCHECK_RUN_ON(&network_sequence_checker_);
// TODO(https://bugs.webrtc.org/9974): Consider removing this workaround.
int64_t time_now_ms = rtc::TimeMillis();
if (last_complete_frame_time_ms_ > 0 &&
time_now_ms - last_complete_frame_time_ms_ > kInactiveStreamThresholdMs) {
frame_buffer_->Clear();
}
last_complete_frame_time_ms_ = time_now_ms;
const PlayoutDelay& playout_delay = frame->EncodedImage().playout_delay_;
if (playout_delay.min_ms >= 0) {
rtc::CritScope cs(&playout_delay_lock_);
frame_minimum_playout_delay_ms_ = playout_delay.min_ms;
UpdatePlayoutDelays();
}
if (playout_delay.max_ms >= 0) {
rtc::CritScope cs(&playout_delay_lock_);
frame_maximum_playout_delay_ms_ = playout_delay.max_ms;
UpdatePlayoutDelays();
}
int64_t last_continuous_pid = frame_buffer_->InsertFrame(std::move(frame)); // std::unique_ptr<video_coding::FrameBuffer> frame_buffer_;
if (last_continuous_pid != -1)
rtp_video_stream_receiver_.FrameContinuous(last_continuous_pid);
}
int64_t FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) {
TRACE_EVENT0("webrtc", "FrameBuffer::InsertFrame");
RTC_DCHECK(frame);
rtc::CritScope lock(&crit_);
const VideoLayerFrameId& id = frame->id;
int64_t last_continuous_picture_id =
!last_continuous_frame_ ? -1 : last_continuous_frame_->picture_id;
if (!ValidReferences(*frame)) {
RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
<< id.picture_id << ":"
<< static_cast<int>(id.spatial_layer)
<< ") has invalid frame references, dropping frame.";
return last_continuous_picture_id;
}
if (frames_.size() >= kMaxFramesBuffered) {
if (frame->is_keyframe()) {
RTC_LOG(LS_WARNING) << "Inserting keyframe (picture_id:spatial_id) ("
<< id.picture_id << ":"
<< static_cast<int>(id.spatial_layer)
<< ") but buffer is full, clearing"
<< " buffer and inserting the frame.";
ClearFramesAndHistory();
} else {
RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
<< id.picture_id << ":"
<< static_cast<int>(id.spatial_layer)
<< ") could not be inserted due to the frame "
<< "buffer being full, dropping frame.";
return last_continuous_picture_id;
}
}
auto last_decoded_frame = decoded_frames_history_.GetLastDecodedFrameId();
auto last_decoded_frame_timestamp =
decoded_frames_history_.GetLastDecodedFrameTimestamp();
if (last_decoded_frame && id <= *last_decoded_frame) {
if (AheadOf(frame->Timestamp(), *last_decoded_frame_timestamp) &&
frame->is_keyframe()) {
// If this frame has a newer timestamp but an earlier picture id then we
// assume there has been a jump in the picture id due to some encoder
// reconfiguration or some other reason. Even though this is not according
// to spec we can still continue to decode from this frame if it is a
// keyframe.
RTC_LOG(LS_WARNING)
<< "A jump in picture id was detected, clearing buffer.";
ClearFramesAndHistory();
last_continuous_picture_id = -1;
} else {
RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
<< id.picture_id << ":"
<< static_cast<int>(id.spatial_layer)
<< ") inserted after frame ("
<< last_decoded_frame->picture_id << ":"
<< static_cast<int>(last_decoded_frame->spatial_layer)
<< ") was handed off for decoding, dropping frame.";
return last_continuous_picture_id;
}
}
// Test if inserting this frame would cause the order of the frames to become
// ambiguous (covering more than half the interval of 2^16). This can happen
// when the picture id make large jumps mid stream.
if (!frames_.empty() && id < frames_.begin()->first &&
frames_.rbegin()->first < id) {
RTC_LOG(LS_WARNING)
<< "A jump in picture id was detected, clearing buffer.";
ClearFramesAndHistory();
last_continuous_picture_id = -1;
}
auto info = frames_.emplace(id, FrameInfo()).first;
if (info->second.frame) {
RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
<< id.picture_id << ":"
<< static_cast<int>(id.spatial_layer)
<< ") already inserted, dropping frame.";
return last_continuous_picture_id;
}
if (!UpdateFrameInfoWithIncomingFrame(*frame, info))
return last_continuous_picture_id;
if (!frame->delayed_by_retransmission())
timing_->IncomingTimestamp(frame->Timestamp(), frame->ReceivedTime()); //注意这里
if (stats_callback_ && IsCompleteSuperFrame(*frame)) {
stats_callback_->OnCompleteFrame(frame->is_keyframe(), frame->size(), //注意这里
frame->contentType());
}
info->second.frame = std::move(frame);
if (info->second.num_missing_continuous == 0) {
info->second.continuous = true;
PropagateContinuity(info);
last_continuous_picture_id = last_continuous_frame_->picture_id;
// Since we now have new continuous frames there might be a better frame
// to return from NextFrame.
new_continuous_frame_event_.Set(); //注意这里
if (callback_queue_) {
callback_queue_->PostTask([this] {// 在单独的线程中进行处理
rtc::CritScope lock(&crit_);
if (!callback_task_.Running())
return;
RTC_CHECK(frame_handler_);
callback_task_.Stop();
StartWaitForNextFrameOnQueue();
});
}
}
return last_continuous_picture_id;
}
void FrameBuffer::StartWaitForNextFrameOnQueue() {
RTC_DCHECK(callback_queue_);
RTC_DCHECK(!callback_task_.Running());
int64_t wait_ms = FindNextFrame(clock_->TimeInMilliseconds());
callback_task_ = RepeatingTaskHandle::DelayedStart(
callback_queue_->Get(), TimeDelta::ms(wait_ms), [this] {
// If this task has not been cancelled, we did not get any new frames
// while waiting. Continue with frame delivery.
rtc::CritScope lock(&crit_);
if (!frames_to_decode_.empty()) {
// We have frames, deliver!
frame_handler_(absl::WrapUnique(GetNextFrame()), kFrameFound);
CancelCallback();
return TimeDelta::Zero(); // Ignored.
} else if (clock_->TimeInMilliseconds() >= latest_return_time_ms_) {
// We have timed out, signal this and stop repeating.
frame_handler_(nullptr, kTimeout);
CancelCallback();
return TimeDelta::Zero(); // Ignored.
} else {
// If there's no frames to decode and there is still time left, it
// means that the frame buffer was cleared between creation and
// execution of this task. Continue waiting for the remaining time.
int64_t wait_ms = FindNextFrame(clock_->TimeInMilliseconds());
return TimeDelta::ms(wait_ms);
}
});
}接收到的视频数据会被放入到 VideoReceiveStream 中的 frame_buffer_ 中；同时呢 VideoReceiveStream 也有一个单独的解码线程 decode_queue_ 不停的从 frame_buffer_ 读取数据进行解码。
扫一扫
926
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
