c语言指针ijk,android ijkplayer c层分析-渲染显示线程

在prepare的stream_open过程中，不仅会启动read_thread，而且会启动video_refresh_thread。今天就来看看这个video_refresh_thread干了什么。

static int video_refresh_thread(void *arg)

{

FFPlayer *ffp = arg;

VideoState *is = ffp->is;

double remaining_time = 0.0;

while (!is->abort_request) {

if (remaining_time > 0.0)

av_usleep((int)(int64_t)(remaining_time * 1000000.0));

remaining_time = REFRESH_RATE;

if (is->show_mode != SHOW_MODE_NONE && (!is->paused || is->force_refresh))

video_refresh(ffp, &remaining_time);

}

return 0;

}

非暂停或强制刷新的时候，循环调用video_refresh。

static void video_refresh(FFPlayer *opaque, double *remaining_time)

{

......

if (!ffp->display_disable && is->show_mode != SHOW_MODE_VIDEO && is->audio_st) {

time = av_gettime_relative() / 1000000.0;

if (is->force_refresh || is->last_vis_time + ffp->rdftspeed < time) {

video_display2(ffp);

is->last_vis_time = time;

}

*remaining_time = FFMIN(*remaining_time, is->last_vis_time + ffp->rdftspeed - time);

}

......

}

video_display2的调用较为关键，之前有外部时钟同步和一些时间检测等。video_display2里直接调用了video_image_display2。

static void video_image_display2(FFPlayer *ffp)

{

VideoState *is = ffp->is;

Frame *vp;

Frame *sp = NULL;

vp = frame_queue_peek_last(&is->pictq);

int latest_seek_load_serial = __atomic_exchange_n(&(is->latest_seek_load_serial), -1, memory_order_seq_cst);

if (latest_seek_load_serial == vp->serial)

ffp->stat.latest_seek_load_duration = (av_gettime() - is->latest_seek_load_start_at) / 1000;

if (vp->bmp) {

if (is->subtitle_st) {

if (frame_queue_nb_remaining(&is->subpq) > 0) {

sp = frame_queue_peek(&is->subpq);

if (vp->pts >= sp->pts + ((float) sp->sub.start_display_time / 1000)) {

if (!sp->uploaded) {

if (sp->sub.num_rects > 0) {

char buffered_text[4096];

if (sp->sub.rects[0]->text) {

strncpy(buffered_text, sp->sub.rects[0]->text, 4096);

}

else if (sp->sub.rects[0]->ass) {

parse_ass_subtitle(sp->sub.rects[0]->ass, buffered_text);

}

ffp_notify_msg4(ffp, FFP_MSG_TIMED_TEXT, 0, 0, buffered_text, sizeof(buffered_text));

}

sp->uploaded = 1;

}

}

}

}

SDL_VoutDisplayYUVOverlay(ffp->vout, vp->bmp);

ffp->stat.vfps = SDL_SpeedSamplerAdd(&ffp->vfps_sampler, FFP_SHOW_VFPS_FFPLAY, "vfps[ffplay]");

if (!ffp->first_video_frame_rendered) {

ffp->first_video_frame_rendered = 1;

ffp_notify_msg1(ffp, FFP_MSG_VIDEO_RENDERING_START);

}

}

}

frame_queue_peek_last获取队列最后一帧，如果帧中的图像正常，继续走sdl渲染yup画面SDL_VoutDisplayYUVOverlay。最后通过消息通知开始渲染ffp_notify_msg1。

下面我们回到video_refresh，看看音画同步的问题是如何处理的：

double last_duration, duration, delay;

Frame *vp, *lastvp;

/* dequeue the picture */

lastvp = frame_queue_peek_last(&is->pictq);

vp = frame_queue_peek(&is->pictq);

if (vp->serial != is->videoq.serial) {

frame_queue_next(&is->pictq);

goto retry;

}

if (lastvp->serial != vp->serial)

is->frame_timer = av_gettime_relative() / 1000000.0;

if (is->paused)

goto display;

/* compute nominal last_duration */

last_duration = vp_duration(is, lastvp, vp);

delay = compute_target_delay(ffp, last_duration, is);

time= av_gettime_relative()/1000000.0;

if (isnan(is->frame_timer) || time < is->frame_timer)

is->frame_timer = time;

if (time < is->frame_timer + delay) {

*remaining_time = FFMIN(is->frame_timer + delay - time, *remaining_time);

goto display;

}

is->frame_timer += delay;

if (delay > 0 && time - is->frame_timer > AV_SYNC_THRESHOLD_MAX)

is->frame_timer = time;

SDL_LockMutex(is->pictq.mutex);

if (!isnan(vp->pts))

update_video_pts(is, vp->pts, vp->pos, vp->serial);

SDL_UnlockMutex(is->pictq.mutex);

if (frame_queue_nb_remaining(&is->pictq) > 1) {

Frame *nextvp = frame_queue_peek_next(&is->pictq);

duration = vp_duration(is, vp, nextvp);

if(!is->step && (ffp->framedrop > 0 || (ffp->framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) && time > is->frame_timer + duration) {

frame_queue_next(&is->pictq);

goto retry;

}

}

首先取出上一帧(lastvp)和当前帧(vp)，然后有个判断是看从video的队列中的序列是否与当前的帧是相同的，如果不是挨个查找下帧，然后跳转到retry，再次执行。我理解的是从FrameQueue队列中找到播放序列相同的这一帧，然后进行后续的操作。下面就是vp_duration了，这里做了一个减法，计算出了这一帧持续的时间。那么后面的delay = compute_target_delay(ffp, last_duration, is);有什么作用呢？

static double compute_target_delay(FFPlayer *ffp, double delay, VideoState *is)

{

double sync_threshold, diff = 0;

/* update delay to follow master synchronisation source */

if (get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER) {

/* if video is slave, we try to correct big delays by

duplicating or deleting a frame */

diff = get_clock(&is->vidclk) - get_master_clock(is);

/* skip or repeat frame. We take into account the

delay to compute the threshold. I still don't know

if it is the best guess */

sync_threshold = FFMAX(AV_SYNC_THRESHOLD_MIN, FFMIN(AV_SYNC_THRESHOLD_MAX, delay));

/* -- by bbcallen: replace is->max_frame_duration with AV_NOSYNC_THRESHOLD */

if (!isnan(diff) && fabs(diff) < AV_NOSYNC_THRESHOLD) {

if (diff <= -sync_threshold)

delay = FFMAX(0, delay + diff);

else if (diff >= sync_threshold && delay > AV_SYNC_FRAMEDUP_THRESHOLD)

delay = delay + diff;

else if (diff >= sync_threshold)

delay = 2 * delay;

}

}

if (ffp) {

ffp->stat.avdelay = delay;

ffp->stat.avdiff = diff;

}

#ifdef FFP_SHOW_AUDIO_DELAY

av_log(NULL, AV_LOG_TRACE, "video: delay=%0.3f A-V=%f\n",

delay, -diff);

#endif

return delay;

}

首先是一个AV_SYNC_VIDEO_MASTER的判断，如果发现主时钟不是video，那么计算当前视频时钟与主时钟的差值，diff = get_clock(&is->vidclk) - get_master_clock(is);计算视频时钟与当前的音频时钟之间的差值。然后根据根据偏差的范围进行了调整延迟时间(视频比音频快，加大下一帧的渲染时间，否则缩短时间)，这里有3种情况判断：如果当前视频时间落后于主时钟，需要减小下一帧画面的等待时间；如果视频帧超前了，并且显示时间大于一个阈值(AV_SYNC_FRAMEDUP_THRESHOLD)，则显示下一帧的时间为超前的时间差加上上一帧的显示时间；如果视频超前了，并且上一帧的显示时间小于这个阈值，则加倍延时。之后设置到了ffp->stat中，并返回。这个返回的偏差值就是后面进行是否抛帧或sleep的判断依据。

回到video_refresh，is->frame_timer = av_gettime_relative() / 1000000.0;这里的frame_time实际上就是上一帧显示的时间，is->frame_timer + delay其实就是当前这一帧显示的时间。那么看这个判断：

if (time < is->frame_timer + delay) {

*remaining_time = FFMIN(is->frame_timer + delay - time, *remaining_time);

goto display;

}

如果播放的时间还没有到达当前这一帧的显示时间，那么直接跳到display，在display中将is->force_refresh变量为0，不显示当前帧,

这里的意思大约是还没到播放的时间，那么跳转出去到display，然后函数就返回了。那么这个函数的调用者video_refresh_thread里面可是循环调用的，所以后面会经过sleep然后再次调用到这个里面。好吧，这里我就简单的理解为如果没有到达显示的时间点，就sleep。后面是如果time - is->frame_timer + delay超过了AV_SYNC_THRESHOLD_MAX，就将is->frame_timer设置为当前时间。这里的目的还不是特别清楚，不过可以感觉到，是为了后面要做判断，估计是要对一些延迟比较高的帧考虑抛帧处理吧。

后面进行update_video_pts更新pts。再下面的这里

if (frame_queue_nb_remaining(&is->pictq) > 1) {

Frame *nextvp = frame_queue_peek_next(&is->pictq);

duration = vp_duration(is, vp, nextvp);

if(!is->step && (ffp->framedrop > 0 || (ffp->framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) && time > is->frame_timer + duration) {

frame_queue_next(&is->pictq);

goto retry;

}

}

如果视频还有下一帧，则拿出下一帧来，计算差值。下面要注意：time > is->frame_timer + duration的判断，如果下一帧的时间点比当前的系统时间慢，也就是说不仅当前的慢了，下一帧的也慢了，慢了2帧，那么久触发丢帧，丢掉当前帧。通过frame_queue_next将缓存游标挪到下一帧，然后goto到retry，重新进行上面的渲染判断。