tutorial 5 音视频同步初印象

最新推荐文章于 2022-06-26 18:02:50 发布

titer1

最新推荐文章于 2022-06-26 18:02:50 发布

阅读量1.4k

点赞数

分类专栏： ffmpeg FFmpeg入门实践与分析

本文链接：https://blog.csdn.net/titer1/article/details/28899065

版权

ffmpeg 同时被 2 个专栏收录

7 篇文章 0 订阅

订阅专栏

FFmpeg入门实践与分析

7 篇文章 4 订阅

订阅专栏

同步视频基础

时间戳

dts解码时间戳

pts显示时间戳

动手实践目标

想不想动手实践看看pts第一次被赋值。
重点研究那个video_fresh-timer函数
鉴于分析的视频只有12帧，
- 理解actual_delay 期望值
- Audio的时间
- 理解一下那个时间戳 vs音频时间的言论
video_fresh-timer关注delay :（现在的时间戳-上一个时间戳）同步阈值两个变量
提前告知 tutorial同步音频到视频，
认识校正方法，是修正delay,要么加倍延迟，要么没有延迟。下一节对于音频来说，要么插值，要么删减？一句话，同步的手段各有。

表面变化：代码变化（tutorial 4 vs tutorial 5）

增加的宏如下：

#define AV_SYNC_THRESHOLD 0.010 //minimum sync time

#define AV_NOSYNC_THRESHOLD 10 //the differ of video and audio exceeds the time, do not sync with audio

值得注意的数据结构变化如下：

struct VideoState{
...
 
double video_clock;

double audio_clock;

double frame_last_pts;
double frame_last_delay;
double frame_timer;

} VideoState;

关于音频当前时钟字段，暂时忽略，详见官方tutorial

视频线程代码及变迁

int video_thread(void *arg) {

VideoState *is = (VideoState *) arg;

AVPacket pkt1, *packet = &pkt1;

int len1, frameFinished;

AVFrame *pFrame;

double pts = 0;

pFrame = avcodec_alloc_frame();

for (;;) {

if (packet_queue_get(&is->videoq, packet, 1) < 0) {

// means we quit getting packets

break;

}

pts = 0;

//save global pts to be stored in pFrame in first call

global_video_pkt_pts = packet->pts;

// printf("packet->pts %d\n", packet->pts );

// printf("packet->dts %d\n", packet->dts );

// Decode video frame

len1 = avcodec_decode_video2(is->video_st->codec, pFrame,

&frameFinished, packet);

if (packet->dts == AV_NOPTS_VALUE && pFrame->opaque

&& *(uint64_t*) pFrame->opaque != AV_NOPTS_VALUE) {

pts = *(uint64_t*) pFrame->opaque;

} else if (packet->dts != AV_NOPTS_VALUE) {

pts = packet->dts;

} else {

pts = 0;

}

// printf("pts %d\n", pts );

pts *= av_q2d(is->video_st->time_base);




// Did we get a video frame?

if (frameFinished) {

pts = synchronize_video(is, pFrame, pts);

if (queue_picture(is, pFrame, pts) < 0) {

break;

}

}

av_free_packet(packet);

}

av_free(pFrame);

return 0;

}

差别有如下：

Pts值的初始化,详细如下图

图中红色部分是额外添加的dts处理

下图中表明还有视频同步函数的加入。

以上是所有的两处大的修改。

视频刷新定时器video_refresh_timer

一句话：请看英文解释，一次性解释清楚

void video_refresh_timer(void *userdata) {

VideoState *is = (VideoState *) userdata;

VideoPicture *vp;

double actual_delay, delay, sync_threshold, ref_clock, diff;

if (is->video_st) {

if (is->pictq_size == 0) {

schedule_refresh(is, 1);

} else {

vp = &is->pictq[is->pictq_rindex];

/* Timing code goes here */

delay = vp->pts - is->frame_last_pts; //the pts from last

if (delay <= 0 || delay >= 1.0) {

//if incorrect delay, use previous one过大或者多小，超过1s,不过理喻

delay = is->frame_last_delay;

}

/* save for next time */

is->frame_last_delay = delay;

is->frame_last_pts = vp->pts;

/* update delay to sync to audio */

ref_clock = get_audio_clock(is);

diff = vp->pts - ref_clock;

/* Skip or repeat the frame. Take delay into account

FFPlay still doesn't "know if this is the best guess." */

sync_threshold = (delay > AV_SYNC_THRESHOLD) ? delay

: AV_SYNC_THRESHOLD;

if (fabs(diff) < AV_NOSYNC_THRESHOLD) {

if (diff <= -sync_threshold) {

delay = 0;

//video is slow, do not delay，视频已太慢（视频预测时间-当前时间都是负值，请赶紧播）

} else if (diff >= sync_threshold)

delay = 2 * delay;

//video is fast, delay double，视频太快，刷新时间变慢点

}

is->frame_timer += delay;

/* computer the REAL delay */

actual_delay = is->frame_timer - (av_gettime() / 1000000.0);

if (actual_delay < 0.010) {

/* Really it should skip the picture instead */

actual_delay = 0.010;

}

schedule_refresh(is, (int) (actual_delay * 1000 + 0.5));

// schedule_refresh(is, 80);

/* show the picture! */

video_display(is);

/* update queue for next picture! */

后续略

}

实验结果分析，

Video_thread代码：

所有亮点都在颜色里。

Video_thread{

...

if (packet->dts == AV_NOPTS_VALUE && pFrame->opaque

&& *(uint64_t*) pFrame->opaque != AV_NOPTS_VALUE) {

pts = *(uint64_t*) pFrame->opaque;

} else if (packet->dts != AV_NOPTS_VALUE) {

pts = packet->dts;//12帧测试案列执行此代码覆盖

} else {

pts = 0;

}

pts *= av_q2d(is->video_st->time_base);

// Did we get a video frame?

if (frameFinished) {

pts = synchronize_video(is, pFrame, pts);

//除第一帧没有pts赋值，其他帧都有赋值。其他备注本案例没有重复帧，

if (queue_picture(is, pFrame, pts) < 0) {

break;

}

}

}

video_refresh_timer代码运行结果

这里的数据结果较多，尤其替换了上一小节定时刷新，值得反复体会。

void video_refresh_timer(void *userdata) {

...

if (is->video_st) {

if (is->pictq_size == 0) {

} else {

vp = &is->pictq[is->pictq_rindex];

/* Timing code goes here */

delay = vp->pts - is->frame_last_pts; //the pts from last

//实验结果非常理想，所有12帧处理的结果都是1.0

if (delay <= 0 || delay >= 1.0) {

//if incorrect delay, use previous one

delay= is->frame_last_delay;

}

/* save for next time */

is->frame_last_delay = delay;

is->frame_last_pts = vp->pts;

/* update delay to sync to audio */

ref_clock = get_audio_clock(is);

diff = vp->pts - ref_clock;//具体的值见下面的表格

/* Skip or repeat the frame. Take delay into account

FFPlay still doesn't "know if this is the best guess." */

sync_threshold = (delay > AV_SYNC_THRESHOLD) ? delay

: AV_SYNC_THRESHOLD;

if (fabs(diff) < AV_NOSYNC_THRESHOLD) {

if (diff <= -sync_threshold) {

delay = 0; //video is slow, do not delay //实验结果非常理想，都是说明视频太慢，

} else if (diff >= sync_threshold) {

delay = 2 * delay; //video is fast, delay double

}

}

is->frame_timer += delay;//鉴于12帧delay=0，这个值就没有增长过

/* computer the REAL delay */

actual_delay = is->frame_timer - (av_gettime() / 1000000.0);

if (actual_delay < 0.010) {//详细的真实delay数据如下，可见都是大大小于阈值

/* Really it should skip the picture instead */

actual_delay = 0.010;

}

logtime=actual_delay * 1000 + 0.5;

schedule_refresh(is, (int) (actual_delay * 1000 + 0.5));//所有填入的时间都是10.5

// schedule_refresh(is, 80);

/* show the picture! */

video_display(is);

//ingore the the other code

}

} else {

schedule_refresh(is, 100);

}

}

12帧的时延分析

表格，对于所有12帧的delay初步结果分析，可以看到所有的帧都太慢，不执行delay. 但是视觉上看来，视频帧还是太慢，这说明音频还得同视频同步，期待下一小节。

// 10 //the differ of video and audio exceeds the time, do not sync with audio

// if(diff <= -sync_threshold)所有出现的sync_threshold都是0.04

diff=-2.0393626991565137, //video is slow, do not delay，

diff=-5.8978444236176193, //video is slow, do not delay，

diff=-9.8762886597938149, //video is slow, do not delay，

diff=-8.9662605435801321, //video is slow, do not delay，

diff=-7.9662605435801321, //video is slow, do not delay，

diff=-6.9662605435801321, //video is slow, do not delay，

diff=-5.9662605435801321, //video is slow, do not delay，

diff=-4.9662605435801321, //video is slow, do not delay，

diff=-3.9662605435801321, //video is slow, do not delay，

diff=-2.9662605435801321, //video is slow, do not delay，

diff=-1.9662605435801321, //video is slow, do not delay，

diff=-0.96626054358013214, //video is slow, do not delay

在计算真实的delay时，涉及到的表格如下，

actual_delay= -3.2161841392517090 ,frame_timer= 1401626707.2479169

actual_delay= -8.3304760456085205 ,frame_timer= 1401626707.2479169

actual_delay= -13.639780044555664 ,frame_timer= 1401626707.2479169

actual_delay= -14.218813180923462 ,frame_timer= 1401626707.2479169

actual_delay= -14.572834014892578 ,frame_timer= 1401626707.2479169

actual_delay= -14.932854175567627 ,frame_timer= 1401626707.2479169

actual_delay= -15.305875062942505 ,frame_timer= 1401626707.2479169

actual_delay= -15.662896156311035 ,frame_timer= 1401626707.2479169

actual_delay= -16.020915985107422 ,frame_timer= 1401626707.2479169

actual_delay= -16.381937026977539 ,frame_timer= 1401626707.2479169

actual_delay= -16.737957000732422 ,frame_timer= 1401626707.2479169

actual_delay= -16.947968959808350 ,frame_timer= 1401626707.2479169

其他：

当播放结束，没有额外的多调用刷屏函数，不像上一个版本。

结果中，也有多线程的随机性出现，情况在预料之中。

未完成分析日志log的一轮起始，下次补充就是。

现在认识依然很浅，比如i b b p的认识

小结：

逻辑很复杂，测试用例很理想，入门在路上。

不要忘了英文注释

不要忘了。。

视频即使同步了，也比音频慢！！

回顾，主要是看vedio_thread ,和播放的timer

附件完整log分析

简化版（没有带修正的log）

Function: video_thread(void *), [pts 2]=0

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=0.00000000000000000

Function: video_refresh_timer(void *), pts之delay=0.00000000000000000

Function: video_refresh_timer(void *), diff=-2.0393626991565137, //video is slow, do not delay

Function: video_refresh_timer(void *),【core time】=10.500000000000000

Function: video_thread(void *), [pts 2]=1

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=1.0000000000000000

Function: video_refresh_timer(void *), pts之delay=1.0000000000000000

Function: video_refresh_timer(void *), diff=-5.8978444236176193, //video is slow, do not delay

Function: video_refresh_timer(void *),【core time】=10.500000000000000

Function: video_thread(void *), [pts 2]=2

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=2.0000000000000000

Function: video_refresh_timer(void *), pts之delay=1.0000000000000000

Function: video_refresh_timer(void *), diff=-9.8762886597938149, //video is slow, do not delay

Function: video_refresh_timer(void *),【core time】=10.500000000000000

Function: video_thread(void *), [pts 2]=3

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=3.0000000000000000

Function: video_thread(void *), [pts 2]=4

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=4.0000000000000000

Function: video_refresh_timer(void *), pts之delay=1.0000000000000000

Function: video_refresh_timer(void *), diff=-8.9662605435801321, //video is slow, do not delay

Function: video_refresh_timer(void *),【core time】=10.500000000000000

Function: video_thread(void *), [pts 2]=5

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=5.0000000000000000

Function: video_refresh_timer(void *), pts之delay=1.0000000000000000

Function: video_refresh_timer(void *), diff=-7.9662605435801321, //video is slow, do not delay

Function: video_refresh_timer(void *),【core time】=10.500000000000000

Function: video_thread(void *), [pts 2]=6

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=6.0000000000000000

Function: video_refresh_timer(void *), pts之delay=1.0000000000000000

Function: video_refresh_timer(void *), diff=-6.9662605435801321, //video is slow, do not delay

Function: video_refresh_timer(void *),【core time】=10.500000000000000

Function: video_thread(void *), [pts 2]=7

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=7.0000000000000000

Function: video_refresh_timer(void *), pts之delay=1.0000000000000000

Function: video_refresh_timer(void *), diff=-5.9662605435801321, //video is slow, do not delay

Function: video_refresh_timer(void *),【core time】=10.500000000000000

Function: video_thread(void *), [pts 2]=8

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=8.0000000000000000

Function: video_refresh_timer(void *), pts之delay=1.0000000000000000

Function: video_refresh_timer(void *), diff=-4.9662605435801321, //video is slow, do not delay

Function: video_refresh_timer(void *),【core time】=10.500000000000000

Function: video_thread(void *), [pts 2]=9

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=9.0000000000000000

Function: video_refresh_timer(void *), pts之delay=1.0000000000000000

Function: video_refresh_timer(void *), diff=-3.9662605435801321, //video is slow, do not delay

Function: video_refresh_timer(void *),【core time】=10.500000000000000

Function: video_thread(void *), [pts 2]=10

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=10.000000000000000

Function: video_refresh_timer(void *), pts之delay=1.0000000000000000

Function: video_refresh_timer(void *), diff=-2.9662605435801321, //video is slow, do not delay

Function: video_refresh_timer(void *),【core time】=10.500000000000000

Function: video_thread(void *), [pts 2]=11

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=11.000000000000000

Function: video_refresh_timer(void *), pts之delay=1.0000000000000000

Function: video_refresh_timer(void *), diff=-1.9662605435801321, //video is slow, do not delay

Function: video_refresh_timer(void *),【core time】=10.500000000000000

Function: video_refresh_timer(void *), pts之delay=1.0000000000000000

Function: video_refresh_timer(void *), diff=-0.96626054358013214, //video is slow, do not delay

Function: video_refresh_timer(void *),【core time】=10.500000000000000

最终用到的log

Function: our_get_buffer(AVCodecContext *, AVFrame *),pts初始化in自定义alloc，pts=0

Function: video_thread(void *), [pts 2]=0

Function: synchronize_video(VideoState *, AVFrame *, double), /* if we aren't given a pts, set it to the clock */, 0.00000000000000000

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=0.00000000000000000

Function: video_refresh_timer(void *), pts之delay=0.00000000000000000

Function: video_refresh_timer(void *), diff=-2.0393626991565137, //video is slow, do not delay，0.040000000000000001 must>=10ms

Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -3.2161841392517090 ,frame_timer= 1401626707.2479169

Function: video_refresh_timer(void *),【core time】=10.500000000000000

Function: video_thread(void *), [pts 2]=1

Function: synchronize_video(VideoState *, AVFrame *, double), /* if we have pts, set video clock to it */ 1.0000000000000000

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=1.0000000000000000

Function: video_refresh_timer(void *), pts之delay=1.0000000000000000

Function: video_refresh_timer(void *), diff=-5.8978444236176193, //video is slow, do not delay，0.040000000000000001 must>=10ms

Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -8.3304760456085205 ,frame_timer= 1401626707.2479169

Function: video_refresh_timer(void *),【core time】=10.500000000000000

Function: video_thread(void *), [pts 2]=2

Function: synchronize_video(VideoState *, AVFrame *, double), /* if we have pts, set video clock to it */ 2.0000000000000000

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=2.0000000000000000

Function: video_refresh_timer(void *), pts之delay=1.0000000000000000

Function: video_refresh_timer(void *), diff=-9.8762886597938149, //video is slow, do not delay，0.040000000000000001 must>=10ms

Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -13.639780044555664 ,frame_timer= 1401626707.2479169

Function: video_refresh_timer(void *),【core time】=10.500000000000000

Function: video_thread(void *), [pts 2]=3

Function: synchronize_video(VideoState *, AVFrame *, double), /* if we have pts, set video clock to it */ 3.0000000000000000

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=3.0000000000000000

Function: video_thread(void *), [pts 2]=4

Function: synchronize_video(VideoState *, AVFrame *, double), /* if we have pts, set video clock to it */ 4.0000000000000000

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=4.0000000000000000

Function: video_refresh_timer(void *), pts之delay=1.0000000000000000

Function: video_refresh_timer(void *), diff=-8.9662605435801321, //video is slow, do not delay，0.040000000000000001 must>=10ms

Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -14.218813180923462 ,frame_timer= 1401626707.2479169

Function: video_refresh_timer(void *),【core time】=10.500000000000000

Function: video_thread(void *), [pts 2]=5

Function: synchronize_video(VideoState *, AVFrame *, double), /* if we have pts, set video clock to it */ 5.0000000000000000

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=5.0000000000000000

Function: video_refresh_timer(void *), pts之delay=1.0000000000000000

Function: video_refresh_timer(void *), diff=-7.9662605435801321, //video is slow, do not delay，0.040000000000000001 must>=10ms

Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -14.572834014892578 ,frame_timer= 1401626707.2479169

Function: video_refresh_timer(void *),【core time】=10.500000000000000

Function: video_thread(void *), [pts 2]=6

Function: synchronize_video(VideoState *, AVFrame *, double), /* if we have pts, set video clock to it */ 6.0000000000000000

Function: video_refresh_timer(void *), pts之delay=1.0000000000000000

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=6.0000000000000000

Function: video_refresh_timer(void *), diff=-6.9662605435801321, //video is slow, do not delay，0.040000000000000001 must>=10ms

Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -14.932854175567627 ,frame_timer= 1401626707.2479169

Function: video_refresh_timer(void *),【core time】=10.500000000000000

Function: video_thread(void *), [pts 2]=7

Function: synchronize_video(VideoState *, AVFrame *, double), /* if we have pts, set video clock to it */ 7.0000000000000000

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=7.0000000000000000

Function: video_refresh_timer(void *), pts之delay=1.0000000000000000

Function: video_refresh_timer(void *), diff=-5.9662605435801321, //video is slow, do not delay，0.040000000000000001 must>=10ms

Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -15.305875062942505 ,frame_timer= 1401626707.2479169

Function: video_refresh_timer(void *),【core time】=10.500000000000000

Function: video_thread(void *), [pts 2]=8

Function: synchronize_video(VideoState *, AVFrame *, double), /* if we have pts, set video clock to it */ 8.0000000000000000

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=8.0000000000000000

Function: video_refresh_timer(void *), pts之delay=1.0000000000000000

Function: video_refresh_timer(void *), diff=-4.9662605435801321, //video is slow, do not delay，0.040000000000000001 must>=10ms

Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -15.662896156311035 ,frame_timer= 1401626707.2479169

Function: video_refresh_timer(void *),【core time】=10.500000000000000

Function: video_thread(void *), [pts 2]=9

Function: synchronize_video(VideoState *, AVFrame *, double), /* if we have pts, set video clock to it */ 9.0000000000000000

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=9.0000000000000000

Function: video_refresh_timer(void *), pts之delay=1.0000000000000000

Function: video_refresh_timer(void *), diff=-3.9662605435801321, //video is slow, do not delay，0.040000000000000001 must>=10ms

Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -16.020915985107422 ,frame_timer= 1401626707.2479169

Function: video_refresh_timer(void *),【core time】=10.500000000000000

Function: video_thread(void *), [pts 2]=10

Function: synchronize_video(VideoState *, AVFrame *, double), /* if we have pts, set video clock to it */ 10.000000000000000

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=10.000000000000000

Function: video_refresh_timer(void *), pts之delay=1.0000000000000000

Function: video_refresh_timer(void *), diff=-2.9662605435801321, //video is slow, do not delay，0.040000000000000001 must>=10ms

Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -16.381937026977539 ,frame_timer= 1401626707.2479169

Function: video_refresh_timer(void *),【core time】=10.500000000000000

Function: video_thread(void *), [pts 2]=11

Function: synchronize_video(VideoState *, AVFrame *, double), /* if we have pts, set video clock to it */ 11.000000000000000

Function: video_thread(void *),【pts 同步后？,经过计算重复帧等】=11.000000000000000

Function: video_refresh_timer(void *), pts之delay=1.0000000000000000

Function: video_refresh_timer(void *), diff=-1.9662605435801321, //video is slow, do not delay，0.040000000000000001 must>=10ms

Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -16.737957000732422 ,frame_timer= 1401626707.2479169

Function: video_refresh_timer(void *),【core time】=10.500000000000000

Function: video_refresh_timer(void *), pts之delay=1.0000000000000000

Function: video_refresh_timer(void *), diff=-0.96626054358013214, //video is slow, do not delay，0.040000000000000001 must>=10ms

Function: video_refresh_timer(void *), /* computer the REAL delay */ actual_delay= -16.947968959808350 ,frame_timer= 1401626707.2479169

Function: video_refresh_timer(void *),【core time】=10.500000000000000

titer1

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
3
评论
tutorial 5 音视频同步初印象

同步视频基础时间戳，dts解码时间戳pts 显示时间戳评价：现在认识依然很浅，比如i b b p的认识动手实践：想不想动手实践看看pts第一次被赋值。重点研究那个video_fresh-timer函数鉴于分析的视频只有12帧，actual_delay expectAudio的时间理解一下那
复制链接

扫一扫