1、音视频同步的实现
音视频同步关键点在计算显示时间:remaining_time;
这个时间是video refresh的sleep时间,默认0.01s,也就是10ms,一刷新,一个fps=24的video,一帧差不多是0.041708s,那么得用4次刷新,但是,第五次怎么办,就得计算remaining_time了,
第五次可以让0.001708s后刷新一次,这个计算是(*remaining_time = FFMIN(is->frame_timer + delay - time, *remaining_time);)取默认值和差值的最小值。
引用:
这段代码的逻辑在上述流程图中有包含。主要思路就是一开始提到的如果视频播放过快,则重复播放上一帧,以等待音频;如果视频播放过慢,则丢帧追赶音频。实现的方式是,参考audio clock,计算上一帧(在屏幕上的那个画面)还应显示多久(含帧本身时长),然后与系统时刻对比,是否该显示下一帧了。
这里与系统时刻的对比,引入了另一个概念——frame_timer。可以理解为帧显示时刻,如更新前,是上一帧的显示时刻;对于更新后(is->frame_timer += delay),则为当前帧显示时刻。
上一帧显示时刻加上delay(还应显示多久(含帧本身时长))即为上一帧应结束显示的时刻。具体原理看如下示意图:
这里给出了3种情况的示意图:
time1:系统时刻小于lastvp结束显示的时刻(frame_timer+dealy),即虚线圆圈位置。此时应该继续显示lastvp
time2:系统时刻大于lastvp的结束显示时刻,但小于vp的结束显示时刻(vp的显示时间开始于虚线圆圈,结束于黑色圆圈)。此时既不重复显示lastvp,也不丢弃vp,即应显示vp
time3:系统时刻大于vp结束显示时刻(黑色圆圈位置,也是nextvp预计的开始显示时刻)。此时应该丢弃vp。
2、丢帧的时机
首先说明得打开丢帧开关framedrop;
计算下一帧显示完的时间和当前系统时间的差值,如果当前系统时间还大于下一帧显示完成时候的时间,那么说明显示慢了,就要丢掉(is->frame_timer + duration- time)。
3、这部分的代码
last_duration = vp_duration(is, lastvp, vp);
/* 2.两帧时间间隔校准:获得校准后的的两帧播放间隔 */
delay = compute_target_delay(last_duration, is);
/* 更新当前系统时间 */
time= av_gettime_relative()/1000000.0;
Lee_LOG("lee video_refresh lastDur:%lf,delay:%lf,relatTime:%lf,is->frame_timer:%lf,is->frame_timer + delay:%lf,diff:%lf ,remaining_time:%lf\n",last_duration,delay,time,is->frame_timer,is->frame_timer + delay,is->frame_timer + delay-time,*remaining_time);
/* 当前系统时间如果小于计算出来的下一帧显示时间,将持续等待 */
//还未到显示的时间,这个时候先保持当前帧的显示,并且计算下次循环的睡眠时间
if (time < is->frame_timer + delay) {
*remaining_time = FFMIN(is->frame_timer + delay - time, *remaining_time);//??toDebug
Lee_LOG("lee video_refresh remaining_time:%lf,is->frame_timer + delay - time:%lf\n",*remaining_time,is->frame_timer + delay - time);
goto display;
}
/* 若已经到达该下一帧的显示时间,则更新显示时间 */
is->frame_timer += delay;
/* 如果当前系统时间大于下一帧渲染时间超过100ms,则直接更新播放时间为当前系统时间 */
if (delay > 0 && time - is->frame_timer > AV_SYNC_THRESHOLD_MAX){
is->frame_timer = time;
}
SDL_LockMutex(is->pictq.mutex);
if (!isnan(vp->pts))
update_video_pts(is, vp->pts, vp->pos, vp->serial);
SDL_UnlockMutex(is->pictq.mutex);
/* 如果还有至少两帧待渲染:计算出下一帧的播放时间 */
if (frame_queue_nb_remaining(&is->pictq) > 1) {
Frame *nextvp = frame_queue_peek_next(&is->pictq);
duration = vp_duration(is, vp, nextvp);
Lee_LOG("lee video_refresh duration:%lf ,time:%lf,is->frame_timer:%lf,diff:%lf\n",duration,time,is->frame_timer,is->frame_timer + duration- time);
// 这个地方要理解下:当前时间time 大于下一帧vp显示完的时间,就要删除vp帧,有可能是设备性能导致的refresh不是0.01s
if(!is->step && (framedrop>0 || (framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) && time > is->frame_timer + duration){
is->frame_drops_late++;
Lee_LOG("lee video_refresh duration:%lf ,time:%lf,is->frame_timer:%lf,diff:%lf,is->frame_drops_late:%d\n",duration,time,is->frame_timer,is->frame_timer + duration- time,is->frame_drops_late);
frame_queue_next(&is->pictq);//删除一帧数据
goto retry;
}
}
4、自己打印的log
看下log,drop帧的,可以看到
差不多相差100ms,差不多过了3帧的时间,所以就刷新慢了。
2021-09-18 15:41:26:178 [140733733279464], [ video_refresh Line#1709] lee video_refresh duration:0.041708 ,time:195240.083831,is->frame_timer:195240.082670,diff:0.040547
2021-09-18 15:41:26:178 [140733733279464], [ video_refresh Line#1756] lee video_refresh frame_queue_next end
2021-09-18 15:41:26:261 [140733733279464], [ video_refresh Line#1713] lee video_refresh duration:0.041708 ,time:195240.167247,is->frame_timer:195240.124378,diff:-0.001160,is->frame_drops_late:7
完整log
2021-09-18 15:41:26:010 [140733733279464], [ video_refresh Line#1683] lee video_refresh lastDur:0.041708,delay:0.041708,relatTime:195239.916450,is->frame_timer:195239.874128,is->frame_timer + delay:195239.915837,diff:-0.000613 ,remaining_time:0.010000
2021-09-18 15:41:26:023 [140733733279464], [ video_refresh Line#1709] lee video_refresh duration:0.041708 ,time:195239.916450,is->frame_timer:195239.915837,diff:0.041095
2021-09-18 15:41:26:023 [140733733279464], [ video_refresh Line#1756] lee video_refresh frame_queue_next end
2021-09-18 15:41:26:026 [140733733279464], [ video_refresh Line#1764] lee video_refresh video_display end
2021-09-18 15:41:26:038 [140733733279464], [ video_refresh Line#1683] lee video_refresh lastDur:0.041708,delay:0.041708,relatTime:195239.944050,is->frame_timer:195239.915837,is->frame_timer + delay:195239.957545,diff:0.013495 ,remaining_time:0.010000
2021-09-18 15:41:26:038 [140733733279464], [ video_refresh Line#1688] lee video_refresh remaining_time:0.010000,is->frame_timer + delay - time:0.013495
2021-09-18 15:41:26:050 [140733733279464], [ video_refresh Line#1683] lee video_refresh lastDur:0.041708,delay:0.041708,relatTime:195239.956333,is->frame_timer:195239.915837,is->frame_timer + delay:195239.957545,diff:0.001212 ,remaining_time:0.010000
2021-09-18 15:41:26:050 [140733733279464], [ video_refresh Line#1688] lee video_refresh remaining_time:0.001212,is->frame_timer + delay - time:0.001212
2021-09-18 15:41:26:053 [140733733279464], [ video_refresh Line#1683] lee video_refresh lastDur:0.041708,delay:0.041708,relatTime:195239.958554,is->frame_timer:195239.915837,is->frame_timer + delay:195239.957545,diff:-0.001009 ,remaining_time:0.010000
2021-09-18 15:41:26:053 [140733733279464], [ video_refresh Line#1709] lee video_refresh duration:0.041708 ,time:195239.958554,is->frame_timer:195239.957545,diff:0.040699
2021-09-18 15:41:26:053 [140733733279464], [ video_refresh Line#1756] lee video_refresh frame_queue_next end
2021-09-18 15:41:26:074 [140733733279464], [ video_refresh Line#1764] lee video_refresh video_display end
2021-09-18 15:41:26:100 [140733733279464], [ video_refresh Line#1683] lee video_refresh lastDur:0.041708,delay:0.041708,relatTime:195240.006408,is->frame_timer:195239.957545,is->frame_timer + delay:195239.999253,diff:-0.007155 ,remaining_time:0.010000
2021-09-18 15:41:26:100 [140733733279464], [ video_refresh Line#1709] lee video_refresh duration:0.041708 ,time:195240.006408,is->frame_timer:195239.999253,diff:0.034554
2021-09-18 15:41:26:100 [140733733279464], [ video_refresh Line#1756] lee video_refresh frame_queue_next end
2021-09-18 15:41:26:132 [140733733279464], [ video_refresh Line#1764] lee video_refresh video_display end
2021-09-18 15:41:26:143 [140733733279464], [ video_refresh Line#1683] lee video_refresh lastDur:0.041708,delay:0.041708,relatTime:195240.048890,is->frame_timer:195239.999253,is->frame_timer + delay:195240.040962,diff:-0.007928 ,remaining_time:0.010000
2021-09-18 15:41:26:143 [140733733279464], [ video_refresh Line#1709] lee video_refresh duration:0.041708 ,time:195240.048890,is->frame_timer:195240.040962,diff:0.033780
2021-09-18 15:41:26:143 [140733733279464], [ video_refresh Line#1756] lee video_refresh frame_queue_next end
2021-09-18 15:41:26:145 [140733733279464], [ video_refresh Line#1764] lee video_refresh video_display end
2021-09-18 15:41:26:156 [140733733279464], [ video_refresh Line#1683] lee video_refresh lastDur:0.041708,delay:0.041708,relatTime:195240.062031,is->frame_timer:195240.040962,is->frame_timer + delay:195240.082670,diff:0.020639 ,remaining_time:0.010000
2021-09-18 15:41:26:156 [140733733279464], [ video_refresh Line#1688] lee video_refresh remaining_time:0.010000,is->frame_timer + delay - time:0.020639
2021-09-18 15:41:26:167 [140733733279464], [ video_refresh Line#1683] lee video_refresh lastDur:0.041708,delay:0.041708,relatTime:195240.072547,is->frame_timer:195240.040962,is->frame_timer + delay:195240.082670,diff:0.010123 ,remaining_time:0.010000
2021-09-18 15:41:26:167 [140733733279464], [ video_refresh Line#1688] lee video_refresh remaining_time:0.010000,is->frame_timer + delay - time:0.010123
2021-09-18 15:41:26:178 [140733733279464], [ video_refresh Line#1683] lee video_refresh lastDur:0.041708,delay:0.041708,relatTime:195240.083831,is->frame_timer:195240.040962,is->frame_timer + delay:195240.082670,diff:-0.001161 ,remaining_time:0.010000
2021-09-18 15:41:26:178 [140733733279464], [ video_refresh Line#1709] lee video_refresh duration:0.041708 ,time:195240.083831,is->frame_timer:195240.082670,diff:0.040547
2021-09-18 15:41:26:178 [140733733279464], [ video_refresh Line#1756] lee video_refresh frame_queue_next end
2021-09-18 15:41:26:248 [140733733279464], [ video_refresh Line#1764] lee video_refresh video_display end
2021-09-18 15:41:26:261 [140733733279464], [ video_refresh Line#1683] lee video_refresh lastDur:0.041708,delay:0.041708,relatTime:195240.167247,is->frame_timer:195240.082670,is->frame_timer + delay:195240.124378,diff:-0.042869 ,remaining_time:0.010000
2021-09-18 15:41:26:261 [140733733279464], [ video_refresh Line#1709] lee video_refresh duration:0.041708 ,time:195240.167247,is->frame_timer:195240.124378,diff:-0.001160
2021-09-18 15:41:26:261 [140733733279464], [ video_refresh Line#1713] lee video_refresh duration:0.041708 ,time:195240.167247,is->frame_timer:195240.124378,diff:-0.001160,is->frame_drops_late:7
2021-09-18 15:41:26:262 [140733733279464], [ video_refresh Line#1683] lee video_refresh lastDur:0.041708,delay:0.041708,relatTime:195240.167481,is->frame_timer:195240.124378,is->frame_timer + delay:195240.166087,diff:-0.001394 ,remaining_time:0.010000
2021-09-18 15:41:26:294 [140733733279464], [ video_refresh Line#1709] lee video_refresh duration:0.041708 ,time:195240.167481,is->frame_timer:195240.166087,diff:0.040314
2021-09-18 15:41:26:295 [140733733279464], [ video_refresh Line#1756] lee video_refresh frame_queue_next end
2021-09-18 15:41:26:309 [140733733279464], [ video_refresh Line#1764] lee video_refresh video_display end
2021-09-18 15:41:26:320 [140733733279464], [ video_refresh Line#1683] lee video_refresh lastDur:0.041708,delay:0.041708,relatTime:195240.225873,is->frame_timer:195240.166087,is->frame_timer + delay:195240.207795,diff:-0.018078 ,remaining_time:0.010000
2021-09-18 15:41:26:320 [140733733279464], [ video_refresh Line#1709] lee video_refresh duration:0.041708 ,time:195240.225873,is->frame_timer:195240.207795,diff:0.023630
2021-09-18 15:41:26:320 [140733733279464], [ video_refresh Line#1756] lee video_refresh frame_queue_next end
2021-09-18 15:41:26:361 [140733733279464], [ video_refresh Line#1764] lee video_refresh video_display end
2021-09-18 15:41:26:409 [140733733279464], [ video_refresh Line#1683] lee video_refresh lastDur:0.041708,delay:0.041708,relatTime:195240.315441,is->frame_timer:195240.207795,is->frame_timer + delay:195240.249503,diff:-0.065938 ,remaining_time:0.010000
2021-09-18 15:41:26:410 [140733733279464], [ video_refresh Line#1709] lee video_refresh duration:0.041708 ,time:195240.315441,is->frame_timer:195240.249503,diff:-0.024229
2021-09-18 15:41:26:410 [140733733279464], [ video_refresh Line#1713] lee video_refresh duration:0.041708 ,time:195240.315441,is->frame_timer:195240.249503,diff:-0.024229,is->frame_drops_late:8
2021-09-18 15:41:26:410 [140733733279464], [ video_refresh Line#1683] lee video_refresh lastDur:0.041708,delay:0.041708,relatTime:195240.315518,is->frame_timer:195240.249503,is->frame_timer + delay:195240.291212,diff:-0.024306 ,remaining_time:0.010000
2021-09-18 15:41:26:410 [140733733279464], [ video_refresh Line#1756] lee video_refresh frame_queue_next end
2021-09-18 15:41:26:418 [140733733279464], [ video_refresh Line#1764] lee video_refresh video_display end
2021-09-18 15:41:26:466 [140733733279464], [ video_refresh Line#1683] lee video_refresh lastDur:0.041708,delay:0.041708,relatTime:195240.371853,is->frame_timer:195240.291212,is->frame_timer + delay:195240.332920,diff:-0.038933 ,remaining_time:0.010000
2021-09-18 15:41:26:466 [140733733279464], [ video_refresh Line#1709] lee video_refresh duration:0.041708 ,time:195240.371853,is->frame_timer:195240.332920,diff:0.002775
2021-09-18 15:41:26:466 [140733733279464], [ video_refresh Line#1756] lee video_refresh frame_queue_next end
2021-09-18 15:41:26:469 [140733733279464], [ video_refresh Line#1764] lee video_refresh video_display end
2021-09-18 15:41:26:526 [140733733279464], [ video_refresh Line#1683] lee video_refresh lastDur:0.041708,delay:0.041708,relatTime:195240.432062,is->frame_timer:195240.332920,is->frame_timer + delay:195240.374628,diff:-0.057434 ,remaining_time:0.010000
2021-09-18 15:41:26:526 [140733733279464], [ video_refresh Line#1709] lee video_refresh duration:0.041708 ,time:195240.432062,is->frame_timer:195240.374628,diff:-0.015725
2021-09-18 15:41:26:526 [140733733279464], [ video_refresh Line#1713] lee video_refresh duration:0.041708 ,time:195240.432062,is->frame_timer:195240.374628,diff:-0.015725,is->frame_drops_late:9
2021-09-18 15:41:26:526 [140733733279464], [ video_refresh Line#1683] lee video_refresh lastDur:0.041708,delay:0.041708,relatTime:195240.432159,is->frame_timer:195240.374628,is->frame_timer + delay:195240.416337,diff:-0.015822 ,remaining_time:0.010000
2021-09-18 15:41:26:526 [140733733279464], [ video_refresh Line#1756] lee video_refresh frame_queue_next end
2021