FFMPEG中的两输入Filter实现(二)

在“FFMPEG中的两输入Filter实现(一)”中分析了滤镜的注册、解析、创建和初始化，这一篇我们就来分析一下 overlay滤镜在ffmpeg中是如何使用的。

下图展示了视频帧从解码到滤波的整体过程，浅紫色部分为滤波实现的主要函数调用关系，整洁起见，一些旁的分支和不太重要的函数没有列出来，会在后面的代码分析中做相应的分析说明。

下面，我们按照上图的调用关系来逐一分析每个重要函数。

1.  send_frame_to_filters():

此函数是将解码得到的一帧视频数据存入当前stream的filter中。

static int send_frame_to_filters(InputStream *ist, AVFrame *decoded_frame)
{
    int i, ret;
    AVFrame *f;

    av_assert1(ist->nb_filters > 0); /* ensure ret is initialized */
    for (i = 0; i < ist->nb_filters; i++) {
        if (i < ist->nb_filters - 1) {
            f = ist->filter_frame;     //InputStream中的filter_frame域，源码中的解释：a ref of decoded_frame, to be sent to filters
            ret = av_frame_ref(f, decoded_frame);   //将解码帧数据复制到ist->filter_frame。
            if (ret < 0)
                break;
        } else
            f = decoded_frame;
        ret = ifilter_send_frame(ist->filters[i], f); //将当前解码帧加入到filter中
        if (ret == AVERROR_EOF)
            ret = 0; /* ignore */
        if (ret < 0) {
            av_log(NULL, AV_LOG_ERROR,
                   "Failed to inject frame into filter network: %s\n", av_err2str(ret));
            break;
        }
    }
    return ret;
}

2. ifilter_send_frame():

接下来我们来分析一下上个函数的主体ifilter_send_frame()，函数体如下：

static int ifilter_send_frame(InputFilter *ifilter, AVFrame *frame)
{
    FilterGraph *fg = ifilter->graph;
    int need_reinit, ret, i;
    //这段是判断InputFilter是否需要重新初始化，当filter中的参数与当前解码帧的参数不一致时，需要对filter重新初始化
    /* determine if the parameters for this input changed */
    need_reinit = ifilter->format != frame->format;
    if (!!ifilter->hw_frames_ctx != !!frame->hw_frames_ctx ||
        (ifilter->hw_frames_ctx && ifilter->hw_frames_ctx->data != frame->hw_frames_ctx->data))
        need_reinit = 1;

    switch (ifilter->ist->st->codecpar->codec_type) {
    case AVMEDIA_TYPE_AUDIO:
        need_reinit |= ifilter->sample_rate    != frame->sample_rate ||
                       ifilter->channels       != frame->channels ||
                       ifilter->channel_layout != frame->channel_layout;
        break;
    case AVMEDIA_TYPE_VIDEO:
        need_reinit |= ifilter->width  != frame->width ||
                       ifilter->height != frame->height;
        break;
    }
//若需要重新初始化参数，则将frame中的相应参数拷贝到ifilter中
    if (need_reinit) {
        ret = ifilter_parameters_from_frame(ifilter, frame);
        if (ret < 0)
            return ret;
    }
//需要重新初始化filter graph，一般在正式转码中需要执行两次，即两个输入的第一帧到来的时候，各执行一次
    /* (re)init the graph if possible, otherwise buffer the frame and return */
    if (need_reinit || !fg->graph) {
        for (i = 0; i < fg->nb_inputs; i++) {
            if (!ifilter_has_all_input_formats(fg)) {
                AVFrame *tmp = av_frame_clone(frame);
                if (!tmp)
                    return AVERROR(ENOMEM);
                av_frame_unref(frame);

                if (!av_fifo_space(ifilter->frame_queue)) {
                    ret = av_fifo_realloc2(ifilter->frame_queue, 2 * av_fifo_size(ifilter->frame_queue));
                    if (ret < 0) {
                        av_frame_free(&tmp);
                        return ret;
                    }
                }
                av_fifo_generic_write(ifilter->frame_queue, &tmp, sizeof(tmp), NULL);
                return 0;
            }
        }

        ret = reap_filters(1);
        if (ret < 0 && ret != AVERROR_EOF) {
            char errbuf[128];
            av_strerror(ret, errbuf, sizeof(errbuf));

            av_log(NULL, AV_LOG_ERROR, "Error while filtering: %s\n", errbuf);
            return ret;
        }

        ret = configure_filtergraph(fg);
        if (ret < 0) {
            av_log(NULL, AV_LOG_ERROR, "Error reinitializing filters!\n");
            return ret;
        }
    }
//函数主体，将解码帧写入到filter中，flag AV_BUFFERSRC_FLAG_PUSH意思是立即输出
    ret = av_buffersrc_add_frame_flags(ifilter->filter, frame, AV_BUFFERSRC_FLAG_PUSH);
    if (ret < 0) {
        av_log(NULL, AV_LOG_ERROR, "Error while filtering\n");
        return ret;
    }

    return 0;

3. av_buffersrc_add_frame_flags():

int attribute_align_arg av_buffersrc_add_frame_flags(AVFilterContext *ctx, AVFrame *frame, int flags)
{
    AVFrame *copy = NULL;
    int ret = 0;
//检查音频声道数，在此例中不会用到
    if (frame && frame->channel_layout &&
        av_get_channel_layout_nb_channels(frame->channel_layout) != av_frame_get_channels(frame)) {
        av_log(ctx, AV_LOG_ERROR, "Layout indicates a different number of channels than actually present\n");
        return AVERROR(EINVAL);
    }
//根据输入的flag参数，决定是否立即输出，从上一个函数我们知道，输出的参数为AV_BUFFERSRC_FLAG_PUSH，因此此处执行立即输出
    if (!(flags & AV_BUFFERSRC_FLAG_KEEP_REF) || !frame)
        return av_buffersrc_add_frame_internal(ctx, frame, flags);
//如果参数不是AV_BUFFERSRC_PUSH，而是例如AV_BUFFERSRC_FLAG_KEEP_REF，则需要将解码帧拷贝一份再输出
    if (!(copy = av_frame_alloc()))
        return AVERROR(ENOMEM);
    ret = av_frame_ref(copy, frame);
    if (ret >= 0)
        ret = av_buffersrc_add_frame_internal(ctx, copy, flags);

    av_frame_free(&copy);
    return ret;
}

4. av_buffersrc_add_frame_internal():

static int av_buffersrc_add_frame_internal(AVFilterContext *ctx,
                                           AVFrame *frame, int flags)
{
    BufferSourceContext *s = ctx->priv;   //将filter实例的私有域赋给BufferSourceContex
    AVFrame *copy;
    int refcounted, ret;

    s->nb_failed_requests = 0;

    if (!frame) {
        s->eof = 1;
        ff_avfilter_link_set_in_status(ctx->outputs[0], AVERROR_EOF, AV_NOPTS_VALUE);
        if ((flags & AV_BUFFERSRC_FLAG_PUSH)) {
            ret = push_frame(ctx->graph);
            if (ret < 0)
                return ret;
        }
        return 0;
    } else if (s->eof)
        return AVERROR(EINVAL);

    refcounted = !!frame->buf[0];  //参考计数ref counted
//格式检查
    if (!(flags & AV_BUFFERSRC_FLAG_NO_CHECK_FORMAT)) {

    switch (ctx->outputs[0]->type) {
    case AVMEDIA_TYPE_VIDEO:
        CHECK_VIDEO_PARAM_CHANGE(ctx, s, frame->width, frame->height,
                                 frame->format);
        break;
    case AVMEDIA_TYPE_AUDIO:
        /* For layouts unknown on input but known on link after negotiation. */
        if (!frame->channel_layout)
            frame->channel_layout = s->channel_layout;
        CHECK_AUDIO_PARAM_CHANGE(ctx, s, frame->sample_rate, frame->channel_layout,
                                 av_frame_get_channels(frame), frame->format);
        break;
    default:
        return AVERROR(EINVAL);
    }

    }
//检查BufferSourceContext中的fifo是否已分配
    if (!av_fifo_space(s->fifo) &&
        (ret = av_fifo_realloc2(s->fifo, av_fifo_size(s->fifo) +
                                         sizeof(copy))) < 0)
        return ret;
//为备份frame分配空间
    if (!(copy = av_frame_alloc()))
        return AVERROR(ENOMEM);
//备份frame到copy
    if (refcounted) {
        av_frame_move_ref(copy, frame);
    } else {
        ret = av_frame_ref(copy, frame);
        if (ret < 0) {
            av_frame_free(&copy);
            return ret;
        }
    }
//将备份frame数据存入BufferSourceContext的fifo结构中
    if ((ret = av_fifo_generic_write(s->fifo, &copy, sizeof(copy), NULL)) < 0) {
        if (refcounted)
            av_frame_move_ref(frame, copy);
        av_frame_free(&copy);
        return ret;
    }
//请求frame，调用的是buffersrc.c中的request_frame()函数，将写入s->fifo的数据读出，并加入到link中，后面会再详细分析该函数
    if ((ret = ctx->output_pads[0].request_frame(ctx->outputs[0])) < 0)
        return ret;
//运行filter并输出一帧
    if ((flags & AV_BUFFERSRC_FLAG_PUSH)) {
        ret = push_frame(ctx->graph);
        if (ret < 0)
            return ret;
    }

    return 0;
}

5. request_frame():

static int request_frame(AVFilterLink *link)
{
    BufferSourceContext *c = link->src->priv;
    AVFrame *frame;
    int ret;

    if (!av_fifo_size(c->fifo)) {
        if (c->eof)
            return AVERROR_EOF;
        c->nb_failed_requests++;
        return AVERROR(EAGAIN);
    }
    av_fifo_generic_read(c->fifo, &frame, sizeof(frame), NULL);  //将BufferSourceContext中的fifo数据，读取一帧出来，存到AVFrame结构体中。

    ret = ff_filter_frame(link, frame);  //将frame加入到link fifo中，并设置filter的优先级

    return ret;
}

6. Push_frame():

static int push_frame(AVFilterGraph *graph)
{
    int ret;

    while (1) {
        ret = ff_filter_graph_run_once(graph);  //运行filter graph
        if (ret == AVERROR(EAGAIN))
            break;
        if (ret < 0)
            return ret;
    }
    return 0;
}

7. ff_filter_graph_run_once():

int ff_filter_graph_run_once(AVFilterGraph *graph)
{
    AVFilterContext *filter;
    unsigned i;

    av_assert0(graph->nb_filters);
    filter = graph->filters[0];            //设置默认filter，此处是overlay
    for (i = 1; i < graph->nb_filters; i++)    //遍历graph中的各个filter，根据优先级确定要执行的filter
        if (graph->filters[i]->ready > filter->ready)
            filter = graph->filters[i];
    if (!filter->ready)
        return AVERROR(EAGAIN);
    return ff_filter_activate(filter);  //执行选定的filter
}

8. ff_filter_activate():

int ff_filter_activate(AVFilterContext *filter)
{
    int ret;

    /* Generic timeline support is not yet implemented but should be easy */
    av_assert1(!(filter->filter->flags & AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC &&
                 filter->filter->activate));
    filter->ready = 0;
    ret = filter->filter->activate ? filter->filter->activate(filter) :   //filter的激活与调度
          ff_filter_activate_default(filter);
    if (ret == FFERROR_NOT_READY)
        ret = 0;
    return ret;
}

9. ff_filter_activate_default():

该函数可能的操作有三种：

(1) 条件满足的情况下，执行ff_filter_frame_to_filter()，输出一帧数据；

(2) 或者，执行forward_status_change()，修改输入状态；

(3) 再或者，运行ff_request_frame_to_filter()来请求一帧数据。

static int ff_filter_activate_default(AVFilterContext *filter)
{
    unsigned i;

    for (i = 0; i < filter->nb_inputs; i++) {
        if (samples_ready(filter->inputs[i], filter->inputs[i]->min_samples)) {
            return ff_filter_frame_to_filter(filter->inputs[i]);    //当frame queue中buffer了足够的数据，则执行filter
        }
    }
    for (i = 0; i < filter->nb_inputs; i++) {
        if (filter->inputs[i]->status_in && !filter->inputs[i]->status_out) {
            av_assert1(!ff_framequeue_queued_frames(&filter->inputs[i]->fifo));
            return forward_status_change(filter, filter->inputs[i]);  //The status change is considered happening after the frames queued in fifo.
        }
    }
    for (i = 0; i < filter->nb_outputs; i++) {
        if (filter->outputs[i]->frame_wanted_out &&      //如果frame_wanted_out非0并且not blocked in，则请求更多数据帧
            !filter->outputs[i]->frame_blocked_in) {
            return ff_request_frame_to_filter(filter->outputs[i]);
        }
    }
    return FFERROR_NOT_READY;
}

10 . ff_filter_frame_to_filter():

static int ff_filter_frame_to_filter(AVFilterLink *link)
{
    AVFrame *frame = NULL;
    AVFilterContext *dst = link->dst;
    int ret;

    av_assert1(ff_framequeue_queued_frames(&link->fifo));
    ret = link->min_samples ?
          ff_inlink_consume_samples(link, link->min_samples, link->max_samples, &frame) :   //从fifo中取一帧数据到frame中
          ff_inlink_consume_frame(link, &frame);
    av_assert1(ret);
    if (ret < 0) {
        av_assert1(!frame);
        return ret;
    }
    /* The filter will soon have received a new frame, that may allow it to
       produce one or more: unblock its outputs. */
    filter_unblock(dst);    //将frame_blocked_in清零
    /* AVFilterPad.filter_frame() expect frame_count_out to have the value
       before the frame; ff_filter_frame_framed() will re-increment it. */
    link->frame_count_out--;
    ret = ff_filter_frame_framed(link, frame);  //执行filter并输出一帧数据
    if (ret < 0 && ret != link->status_out) {
        ff_avfilter_link_set_out_status(link, ret, AV_NOPTS_VALUE);  //设置link的输出status
    } else {
        /* Run once again, to see if several frames were available, or if
           the input status has also changed, or any other reason. */
        ff_filter_set_ready(dst, 300);   //设置输出的优先级
    }
    return ret;
}

11. ff_filter_frame_framed():

static int ff_filter_frame_framed(AVFilterLink *link, AVFrame *frame)
{
    int (*filter_frame)(AVFilterLink *, AVFrame *);
    AVFilterContext *dstctx = link->dst;
    AVFilterPad *dst = link->dstpad;
    int ret;

    if (!(filter_frame = dst->filter_frame))
        filter_frame = default_filter_frame;

    if (dst->needs_writable) {
        ret = ff_inlink_make_frame_writable(link, &frame);
        if (ret < 0)
            goto fail;
    }

    ff_inlink_process_commands(link, frame);
    dstctx->is_disabled = !ff_inlink_evaluate_timeline_at_frame(link, frame);

    if (dstctx->is_disabled &&
        (dstctx->filter->flags & AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC))
        filter_frame = default_filter_frame;
    ret = filter_frame(link, frame);  //函数主体，执行从link fifo中取出来的一帧数据
    link->frame_count_out++;
    return ret;

fail:
    av_frame_free(&frame);
    return ret;
}

12.  filter_frame():

对于overlay filter来说，filter_frame调用的是vf_overlay.c中的filter_frame()函数，函数定义如下：

static int filter_frame(AVFilterLink *inlink, AVFrame *inpicref)
{
    OverlayContext *s = inlink->dst->priv;  //给overlay 实例赋值
    av_log(inlink->dst, AV_LOG_DEBUG, "Incoming frame (time:%s) from link #%d\n", av_ts2timestr(inpicref->pts, &inlink->time_base), FF_INLINK_IDX(inlink));
    return ff_dualinput_filter_frame(&s->dinput, inlink, inpicref);  //执行两输入filter
}

13. ff_dualinput_filter_frame():

继续调用ff_framesync_filter_frame()

int ff_dualinput_filter_frame(FFDualInputContext *s,
                                   AVFilterLink *inlink, AVFrame *in)
{
    return ff_framesync_filter_frame(&s->fs, inlink, in); 
}

14. ff_framesync_filter_frame():

再来看看接下来是个什么鬼。

该函数调用了两次ff_framesync_process_frame()，中间还调用了一次ff_frame_add_frame()。

int ff_framesync_filter_frame(FFFrameSync *fs, AVFilterLink *inlink,
                              AVFrame *in)
{
    int ret;

    if ((ret = ff_framesync_process_frame(fs, 1)) < 0)   //确定请求的帧，即FFFrameSync两输入中缺失的输入
        return ret;
    if ((ret = ff_framesync_add_frame(fs, FF_INLINK_IDX(inlink), in)) < 0)   //将当前帧加入到FFFrameSync结构体
        return ret;
    if ((ret = ff_framesync_process_frame(fs, 0)) < 0)   //对FFFrameSync中的两个输入执行filter操作
        return ret;
    return 0;
}

15. ff_framesync_process_frame():

int ff_framesync_process_frame(FFFrameSync *fs, unsigned all)
{
    int ret, count = 0;

    av_assert0(fs->on_event);
    while (1) {
        ff_framesync_next(fs);  //确定请求的输入；实现帧同步
        if (fs->eof || !fs->frame_ready)
            break;
        if ((ret = fs->on_event(fs)) < 0)   //指向最终filter实现的函数指针
            return ret;
        ff_framesync_drop(fs);   //执行完filter操作后将ready值清零
        count++;
        if (!all)
            break;
    }
    if (!count && fs->eof)
        return AVERROR_EOF;
    return count;
}

16. ff_framesync_next():

void ff_framesync_next(FFFrameSync *fs)
{
    unsigned i;

    av_assert0(!fs->frame_ready);
    for (i = 0; i < fs->nb_in; i++)
        if (!fs->in[i].have_next && fs->in[i].queue.available)
            framesync_inject_frame(fs, i, ff_bufqueue_get(&fs->in[i].queue));  
    fs->frame_ready = 0;      //filter实现之前，将ready值清零
    framesync_advance(fs);   //在该函数中确定request的输入；或实现帧同步
}

17. framesync_advance()：

static void framesync_advance(FFFrameSync *fs)
{
    int latest;
    unsigned i;
    int64_t pts;

    if (fs->eof)
        return;
    while (!fs->frame_ready) {    //若非frame_ready，则确定request的输入index
        latest = -1;
        for (i = 0; i < fs->nb_in; i++) {
            if (!fs->in[i].have_next) {
                if (latest < 0 || fs->in[i].pts < fs->in[latest].pts)
                    latest = i;
            }
        }
        if (latest >= 0) {
            fs->in_request = latest;    //得到in_request值，退出循环并返回
            break;
        }
//下面这段在对两个视频帧间同步要求较高的场景下非常重要，曾经遇到计算转码前后的两个视频的psnr值时，由于pts不对齐，导致计算出来的psnr值非常小。
        pts = fs->in[0].pts_next;
        for (i = 1; i < fs->nb_in; i++)
            if (fs->in[i].pts_next < pts)
                pts = fs->in[i].pts_next;    //取两输入中较小的pts值为基准
        if (pts == INT64_MAX) {
            fs->eof = 1;
            break;
        }
        for (i = 0; i < fs->nb_in; i++) {    //根据最新的pts值更新相应的输入
            if (fs->in[i].pts_next == pts ||
                (fs->in[i].before == EXT_INFINITY &&
                 fs->in[i].state == STATE_BOF)) {
                av_frame_free(&fs->in[i].frame);
                fs->in[i].frame      = fs->in[i].frame_next;
                fs->in[i].pts        = fs->in[i].pts_next;
                fs->in[i].frame_next = NULL;
                fs->in[i].pts_next   = AV_NOPTS_VALUE;
                fs->in[i].have_next  = 0;
                fs->in[i].state      = fs->in[i].frame ? STATE_RUN : STATE_EOF;
                if (fs->in[i].sync == fs->sync_level && fs->in[i].frame)
                    fs->frame_ready = 1;         //设置frame_ready
                if (fs->in[i].state == STATE_EOF &&
                    fs->in[i].after == EXT_STOP)
                    fs->eof = 1;
            }
        }
        if (fs->eof)
            fs->frame_ready = 0;
        if (fs->frame_ready)
            for (i = 0; i < fs->nb_in; i++)
                if ((fs->in[i].state == STATE_BOF &&
                     fs->in[i].before == EXT_STOP))
                    fs->frame_ready = 0;
        fs->pts = pts;     //更新FFFrame结构体的pts为最新的pts
    }
}

18. fs->on_event():

此例中，该函数指针实际调用的是dual_input.c中的process_frame()。

static int process_frame(FFFrameSync *fs)
{
    AVFilterContext *ctx = fs->parent;
    FFDualInputContext *s = fs->opaque;
    AVFrame *mainpic = NULL, *secondpic = NULL;
    int ret = 0;

    if ((ret = ff_framesync_get_frame(&s->fs, 0, &mainpic,   1)) < 0 ||    //获取两输入的主图片
        (ret = ff_framesync_get_frame(&s->fs, 1, &secondpic, 0)) < 0) {    //获取两输入中的辅图片，overlay中即logo图片
        av_frame_free(&mainpic);
        return ret;
    }
    av_assert0(mainpic);
    mainpic->pts = av_rescale_q(s->fs.pts, s->fs.time_base, ctx->outputs[0]->time_base);  //主图片的pts
    if (secondpic && !ctx->is_disabled)
        mainpic = s->process(ctx, mainpic, secondpic);    //通过函数指针调用具体的滤镜处理
    ret = ff_filter_frame(ctx->outputs[0], mainpic);   //将滤波后的视频帧加入输出fifo中，并清掉输出frame_blocked_in和输出filter优先级
    av_assert1(ret != AVERROR(EAGAIN));
    return ret;
}

19. s->process():

本例中，该函数指针指向的是vf_overlay.c中的do_blend()函数，本篇暂不对overlay的算法做进一步分析。

static AVFrame *do_blend(AVFilterContext *ctx, AVFrame *mainpic,
                         const AVFrame *second)
{
    OverlayContext *s = ctx->priv;
    AVFilterLink *inlink = ctx->inputs[0];

    if (s->eval_mode == EVAL_MODE_FRAME) {
        int64_t pos = av_frame_get_pkt_pos(mainpic);

        s->var_values[VAR_N] = inlink->frame_count_out;
        s->var_values[VAR_T] = mainpic->pts == AV_NOPTS_VALUE ?
            NAN : mainpic->pts * av_q2d(inlink->time_base);
        s->var_values[VAR_POS] = pos == -1 ? NAN : pos;

        s->var_values[VAR_OVERLAY_W] = s->var_values[VAR_OW] = second->width;
        s->var_values[VAR_OVERLAY_H] = s->var_values[VAR_OH] = second->height;
        s->var_values[VAR_MAIN_W   ] = s->var_values[VAR_MW] = mainpic->width;
        s->var_values[VAR_MAIN_H   ] = s->var_values[VAR_MH] = mainpic->height;

        eval_expr(ctx);
        av_log(ctx, AV_LOG_DEBUG, "n:%f t:%f pos:%f x:%f xi:%d y:%f yi:%d\n",
               s->var_values[VAR_N], s->var_values[VAR_T], s->var_values[VAR_POS],
               s->var_values[VAR_X], s->x,
               s->var_values[VAR_Y], s->y);
    }

    if (s->x < mainpic->width  && s->x + second->width  >= 0 ||
        s->y < mainpic->height && s->y + second->height >= 0)
        s->blend_image(ctx, mainpic, second, s->x, s->y);
    return mainpic;
}

至此，overlay filter的调用就自顶向下地实现了，有点小复杂，但也是ffmpeg框架整合各种滤镜的实现方式。