给音视频帧打上时间戳之前必须要做一些初始化的操作,I和J版本是一致的,大致过程如下:
if (s->mNumRTCPPacketsReceived++ == 0) {
如果收到的是第一个rtcp包会给MyHandler发送一个消息,用于改变一些变量的状态
sp notify = s->mNotifyMsg->dup();
notify->setInt32("first-rtcp", true);
notify->post();
}
const uint8_t *data = buffer->data();
size_t size = buffer->size();
while (size > 0) {
if (size < 8) {
// Too short to be a valid RTCP header
return -1;
}
if ((data[0] >> 6) != 2) {
// Unsupported version.
return -1;
}
size_t headerLength = 4 * (data[2] << 8 | data[3]) + 4;
if (size < headerLength) {
// Only received a partial packet?
return -1;
}
switch (data[1]) {
case 200:
{
parseSR(s, data, headerLength); 如果是SR类型的包
break;
}
case 203:
{
parseBYE(s, data, headerLength);
break;
}
default:
{
ALOGW("Unknown RTCP packet type %u of size %d",
(unsigned)data[1], headerLength);
break;
}
}
data += headerLength;
size -= headerLength;
}
return OK;
StreamInfo *s, const uint8_t *data, size_t size) {
size_t RC = data[0] & 0x1f;
if (size < (7 + RC * 6) * 4) {
// Packet too short for the minimal SR header.
return -1;
}
uint32_t id = u32at(&data[4]);
uint64_t ntpTime = u64at(&data[8]);
uint32_t rtpTime = u32at(&data[16]);
SR包中最重要的就是携带了上面两个参考时间,后续用于给音视频帧打时间戳的公式中需要用到
ALOGI("XXX timeUpdate: ssrc=0xx, rtpTime %u == ntpTime %.3f",
id,
rtpTime,
(ntpTime >> 32) + (double)(ntpTime & 0xffffffff) / (1ll << 32));
sp source = findSource(s, id);
source->timeUpdate(rtpTime, ntpTime);
return 0;
mLastNTPTime = ntpTime;
mLastNTPTimeUpdateUs = ALooper::GetNowUs();
sp notify = mNotify->dup();
notify->setInt32("time-update", true);
notify->setInt32("rtp-time", rtpTime);
notify->setInt64("ntp-time", ntpTime);
notify->post();
case 'accu':
{
int32_t timeUpdate;
if (msg->findInt32("time-update", &timeUpdate) && timeUpdate) {
size_t trackIndex;
CHECK(msg->findSize("track-index", &trackIndex));
uint32_t rtpTime;
uint64_t ntpTime;
CHECK(msg->findInt32("rtp-time", (int32_t *)&rtpTime));
CHECK(msg->findInt64("ntp-time", (int64_t *)&ntpTime));
onTimeUpdate(trackIndex, rtpTime, ntpTime);
break;
}
void onTimeUpdate(int32_t trackIndex, uint32_t rtpTime, uint64_t ntpTime) {
ALOGV("onTimeUpdate track %d, rtpTime = 0xx, ntpTime = 0x6llx",
trackIndex, rtpTime, ntpTime);
int64_t ntpTimeUs = (int64_t)(ntpTime * 1E6 / (1ll << 32));
判断是音频还是视频轨
TrackInfo *track = &mTracks.editItemAt(trackIndex);
track->mRTPAnchor = rtpTime;
track->mNTPAnchorUs = ntpTimeUs;
设置该音视频轨结构体中的两个参数
if (mNTPAnchorUs < 0) {
mNTPAnchorUs = ntpTimeUs;
mMediaAnchorUs = mLastMediaTimeUs;
}
这里还有一些I版本新增的一些处理,这里暂且先略掉
}
track->mRTPAnchor = rtpTime;
track->mNTPAnchorUs = ntpTimeUs;
mNTPAnchorUs = ntpTimeUs;
mMediaAnchorUs = mLastMediaTimeUs;
bool addMediaTimestamp(
int32_t trackIndex, const TrackInfo *track,
const sp &accessUnit) {
uint32_t rtpTime;
收到的每一帧数据中都会携带一个rtp时间信息
CHECK(accessUnit->meta()->findInt32(
"rtp-time", (int32_t *)&rtpTime));
int64_t relRtpTimeUs =
(((int64_t)rtpTime - (int64_t)track->mRTPAnchor) * 1000000ll)
/ track->mTimeScale;
int64_t ntpTimeUs = track->mNTPAnchorUs + relRtpTimeUs;
int64_t mediaTimeUs = mMediaAnchorUs + ntpTimeUs - mNTPAnchorUs;
if (mediaTimeUs > mLastMediaTimeUs) {
更改当前收到的媒体信息的时间
mLastMediaTimeUs = mediaTimeUs;
}
if (mediaTimeUs < 0) {
ALOGV("dropping early accessUnit.");
return false;
}
ALOGV("track %d rtpTime=%d mediaTimeUs = %lld us (%.2f secs)",
trackIndex, rtpTime, mediaTimeUs, mediaTimeUs / 1E6);
accessUnit->meta()->setInt64("timeUs", mediaTimeUs);
mediaTimeUs就是我们计算出来的这一帧的时间戳,也就是这一帧需要在什么时候播放,最后set到这一帧的属性中去。
return true;
}
(((int64_t)rtpTime - (int64_t)track->mRTPAnchor) * 1000000ll)
track->mTimeScale
(((int64_t)rtpTime - (int64_t)track->mRTPAnchor) * 1000000ll)
-------------------------------------------------------------
track->mTimeScale
(int64_t)rtpTime - 0
-------------------------------------------------------------
track->mTimeScale
case 'tiou':
{
if (!mReceivedFirstRTCPPacket
) {
if (mReceivedFirstRTPPacket && !mTryFakeRTCP) {
ALOGW("We received RTP packets but no RTCP packets, "
"using fake timestamps.");
mTryFakeRTCP = true;
mReceivedFirstRTCPPacket
= true;
fakeTimestamps();
}
void fakeTimestamps() {
mNTPAnchorUs = -1ll;
for (size_t i = 0; i < mTracks.size(); ++i) {
onTimeUpdate(i, 0, 0ll);
}
}
if (addMediaTimestamp(trackIndex, track, accessUnit)) {
postQueueAccessUnit(trackIndex, accessUnit);
}
TrackInfo *info = &mTracks.editItemAt(trackIndex);
sp source = info->mSource;
if (source != NULL) {
uint32_t rtpTime;
CHECK(accessUnit->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
if (!info->mNPTMappingValid) {
// This is a live stream, we didn't receive any normal
// playtime mapping. We won't map to npt time.
source->queueAccessUnit(accessUnit);
break;
}
int64_t nptUs =
((double)rtpTime - (double)info->mRTPTime)
/ info->mTimeScale
* 1000000ll
+ info->mNormalPlaytimeUs;
accessUnit->meta()->setInt64("timeUs", nptUs);
source->queueAccessUnit(accessUnit);
sp source = info->mSource;
if (source != NULL) {
uint32_t rtpTime;
CHECK(accessUnit->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
if (!info->mNPTMappingValid) {
// This is a live stream, we didn't receive any normal
// playtime mapping. Assume the first packets correspond
// to time 0.
LOGV("This is a live stream, assuming time = 0");
info->mRTPTime = rtpTime;
info->mNormalPlaytimeUs = 0ll;
info->mNPTMappingValid = true;
}
int64_t nptUs =
((double)rtpTime - (double)info->mRTPTime)
/ info->mTimeScale
* 1000000ll
+ info->mNormalPlaytimeUs;
accessUnit->meta()->setInt64("timeUs", nptUs);
source->queueAccessUnit(accessUnit);
}
((double)rtpTime - (double)info->mRTPTime)
------------------------------------------- * 1000000ll + info->mNormalPlaytimeUs
info->mTimeScale
根据服务器是否发送SR类型的RTCP包分为两种情况
1:服务器发送SR类型RTCP包
status_t ARTPConnection::parseRTCP(StreamInfo *s, const sp &buffer) {
}
status_t ARTPConnection::parseSR(
#if 0
#endif
}
void ARTPSource::timeUpdate(uint32_t rtpTime, uint64_t ntpTime) {
}
这个消息最终会发送到MyHandler中,处理代码如下
这样当针对音视频轨的两个SR类型的RTCP包都收到后,就会两次进入到onTimeUpdate这个函数中,进行相关参数的设置,其中下面这四个参数最重要了
所谓Anchor也就是描点也就是一个参考点的意思,后续来的音视频帧都会携带一个时间点,然后相对这个描点进行计算,计算该帧的时间戳
下面看给音视频帧计算时间戳的方法,这个是在addMediaTimestamp函数中
上面蓝色粗体部分就是整个的计算公式,公式如下:
track->mNTPAnchorUs + ------------------------------------------------------------- + mMediaAnchorUs - mNTPAnchorUs
正常情况下
track->mNTPAnchorUs -
mNTPAnchorUs =
mMediaAnchorUs = 0
所以最后剩下:
这个公式也就很明朗了,就是用这一帧中携带的rtp时间减去sr包中携带的参考的rtp时间,除以时间幅度就是我们需要的值
2:服务器没有发送SR类型的RTCP包,其实也就是没有发送RTCP包的情况
这种情况后半部分处理是一致的,即公式是一致的,就是一些初始值全部初始化为0了,这样公式就简化为
简单的看下处理流程
每次接受到一个帧的时候到会发送一个timeout的消息,检查10秒后有没有接受到数据,上面这个条件就是判断最初的10秒内有没有收到SR类型的rtcp包
如果没有就调用下面这个函数
最后和第一种流程一致直接两次调用onTimeUpdate函数,不过这次传入的参数不是SR类型的RTCP包中携带的参考时间,而是全部设为0.后面的处理逻辑就一致了。
不过这里的打上的时间戳最后还会被覆盖
在onAccessUnitComplete函数中
最后会在postQueueAccessUnit函数中发给RTSPSource.cpp处理(放到AnotherPacketSource的待解码队列)
不过放进去之前还会做一些处理
看这里重新为这一帧打上了时间戳,然后放到了待解码队列中去了,对于点播的流媒体这个时间戳肯定是会重新打上的,对于直播流媒体J版本就不会打了,直接在内部的if中break了,而对于I版本也是会重新打上的。
I版本代码如下:
这里不跟踪具体跟踪每个变量值是怎么赋值的,直接给出这次打时间戳的方案:
MyHandler中打时间戳都是相对于SR中的时间描点进行计算,而这里是根据音视频到来的第一帧作为参考描点,公式:
这样算出来的音视频第一帧的时间戳肯定是0,后续音视频帧的时间戳也是从0开始慢慢增大。
而MyHandler中相对SR计算出来的时间戳就不一定是(而且大概率)不是从0开始的,而且测试发现,对于直播流媒体,由于没有给出视频的总时间,进度条右端会采用默认的一个小时,而有时间通过SR类型的包计算出来的时间戳最开始就超过了1个小时,这样就导致进度条直接超过了屏幕的显示范围,穿过去了,下次带手机来截个图。