FFmpeg源码分析：AVFilterGraph与AVFilter

FFmpeg在libavfilter模块提供简单与复杂的音视频滤镜，所有滤波器由AVFilterGraph滤波器图表连接起来。简单滤镜为一对一输出，复杂滤镜为多对一输出。重要的结构体包括：AVFilterGraph、AVFilterLink、AVFilterContext、AVFilter。支持在滤波器图表指定位置插入AVFilter滤波器，然后由AVFilterLink把滤波器连接起来。

1、AVFilterGraph

AVFilterGraph为滤波器图表结构体，保存滤波器上下文、滤波器数量、线程类型、线程数量、回调函数等，代码位于libavfilter/avfilter.h头文件中：

typedef struct AVFilterGraph {
    const AVClass *av_class;
	// 滤波器上下文
    AVFilterContext **filters;
	// 滤波器数量
    unsigned nb_filters;

    char *scale_sws_opts;
    int thread_type;
    int nb_threads;
    AVFilterGraphInternal *internal;
    void *opaque;
    // 用户设置的回调函数
    avfilter_execute_func *execute;

    char *aresample_swr_opts;
    AVFilterLink **sink_links;
    int sink_links_count;
    unsigned disable_auto_convert;
} AVFilterGraph;

2、AVFilterLink

AVFilterLink为滤波器的连接器，负责把两个滤波器连接起来，内部包含指向源滤波器和目标滤波器的指针。需要注意的是，应用程序不能直接访问AVFilterLink，使用buffersrc和buffersink的API代替。代码同样位于avfilter.h头文件中：

struct AVFilterLink {
	// 源滤波器上下文
    AVFilterContext *src;
    AVFilterPad *srcpad;
    // 目标滤波器上下文
    AVFilterContext *dst;
    AVFilterPad *dstpad;
    // 媒体类型
    enum AVMediaType type;

    // 视频参数
    int w;
    int h;
    AVRational sample_aspect_ratio;
    // 音频参数
    uint64_t channel_layout;
    int sample_rate;
    int format;
    // pts的时间基
    AVRational time_base;

    AVFilterFormatsConfig incfg;
    AVFilterFormatsConfig outcfg;

    // 连接器的初始化状态
    enum {
        AVLINK_UNINIT = 0, // 没初始化
        AVLINK_STARTINIT,  // 已初始化，没完成
        AVLINK_INIT        // 已完成初始化
    } init_state;

    struct AVFilterGraph *graph;
    // 当前pts时间戳
    int64_t current_pts;
    int64_t current_pts_us;
    int age_index;
    AVRational frame_rate;
    AVFrame *partial_buf;
    int partial_buf_size;
    int min_samples;
    int max_samples;
    int channels;
    int64_t frame_count_in, frame_count_out;
    void *frame_pool;
    int frame_wanted_out;
    AVBufferRef *hw_frames_ctx;

#ifndef FF_INTERNAL_FIELDS

    // 内部数据结构(保留)
    char reserved[0xF000];

#else /* FF_INTERNAL_FIELDS */

    // 滤波器的帧队列
    FFFrameQueue fifo;
    frame_blocked_in;
    int status_in;
    int64_t status_in_pts;
    int status_out;

#endif /* FF_INTERNAL_FIELDS */
};

3、AVFilterContext

AVFilterContext为滤波器上下文的结构体，包含滤波器实例、名字、输入Pad和Link、输出Pad和Link，还有ready字段表示滤波器准备就绪且带有优先级。具体代码如下：

struct AVFilterContext {
    const AVClass *av_class;
    // 滤波器实例
    const AVFilter *filter;
    // 滤波器名字
    char *name;

    AVFilterPad   *input_pads;
    AVFilterLink **inputs;
    unsigned    nb_inputs;
    AVFilterPad   *output_pads;
    AVFilterLink **outputs;
    unsigned    nb_outputs;

    void *priv;
    struct AVFilterGraph *graph;
    int thread_type;
    AVFilterInternal *internal;
    struct AVFilterCommand *command_queue;
    char *enable_str;
    void *enable;
    double *var_values;
    int is_disabled;
    AVBufferRef *hw_device_ctx;
    int nb_threads;

    // 滤波器准备就绪，带优先级
    unsigned ready;
    int extra_hw_frames;
};

4、AVFilter

AVFilter为滤波器的结构体，包含滤波器名字与描述、输入输出的AVFilterPad、滤波器操作的函数指针。具体代码如下：

typedef struct AVFilter {
    // 滤波器名字
    const char *name;
    // 滤波器描述
    const char *description;
    const AVFilterPad *inputs;
    const AVFilterPad *outputs;
    const AVClass *priv_class;
    int flags;
    int priv_size;
    int flags_internal;

    // 预初始化函数指针
    int (*preinit)(AVFilterContext *ctx);
    // 初始化函数指针
    int (*init)(AVFilterContext *ctx);
    // 带字典参数的初始化函数指针
    int (*init_dict)(AVFilterContext *ctx, AVDictionary **options);
    // 反初始化函数指针
    void (*uninit)(AVFilterContext *ctx);
    // 查询滤波器支持的输入输出格式
    int (*query_formats)(AVFilterContext *);
    // 处理命令函数指针
    int (*process_command)(AVFilterContext *, const char *cmd, 
	    const char *arg, char *res, int res_len, int flags);
    // 带透传指针的初始化函数指针
    int (*init_opaque)(AVFilterContext *ctx, void *opaque);
    // 滤波器激活函数指针
    int (*activate)(AVFilterContext *ctx);
} AVFilter;

5、avfilter_graph_create_filter

负责创建并添加滤波器实例，并且添加到AVFilterGraph图表。代码如下：

int avfilter_graph_create_filter(AVFilterContext **filt_ctx, const AVFilter *filt,
                                 const char *name, const char *args, void *opaque,
                                 AVFilterGraph *graph_ctx)
{
    int ret;
    // 分配滤波器上下文
    *filt_ctx = avfilter_graph_alloc_filter(graph_ctx, filt, name);
    if (!*filt_ctx)
        return AVERROR(ENOMEM);
    // 初始化滤波器
    ret = avfilter_init_str(*filt_ctx, args);
    if (ret < 0)
        goto fail;

    return 0;
fail:
    if (*filt_ctx)
        avfilter_free(*filt_ctx);
    *filt_ctx = NULL;
    return ret;
}

调用avfilter_graph_alloc_filter()函数来分配滤波器上下文，调用ff_graph_thread_init()来创建滤波线程，调用ff_filter_alloc()来真正分配滤波器上下文。代码如下：

AVFilterContext *avfilter_graph_alloc_filter(AVFilterGraph *graph,
                                             const AVFilter *filter,
                                             const char *name)
{
    AVFilterContext **filters, *s;
    // 创建滤波器线程
    if (graph->thread_type && !graph->internal->thread_execute) {
        if (graph->execute) {
            graph->internal->thread_execute = graph->execute;
        } else {
            int ret = ff_graph_thread_init(graph);
            if (ret < 0) {
                return NULL;
            }
        }
    }
    // 分配滤波器上下文
    s = ff_filter_alloc(filter, name);
    if (!s)
        return NULL;
    filters = av_realloc(graph->filters, sizeof(*filters) * (graph->nb_filters + 1));
    if (!filters) {
        avfilter_free(s);
        return NULL;
    }
    graph->filters = filters;
    graph->filters[graph->nb_filters++] = s;
    s->graph = graph;

    return s;
}

6、avfilter_init_str

 avfilter_init_str()函数，用于初始化滤波器，包括兼容旧版命令语法与新版命令语法。代码位于avfilter.c中：

int avfilter_init_str(AVFilterContext *filter, const char *args)
{
    AVDictionary *options = NULL;
    AVDictionaryEntry *e;
    int ret = 0;

    if (args && *args) {
        if (!filter->filter->priv_class) {
            return AVERROR(EINVAL);
        }
// 兼容旧版语法
#if FF_API_OLD_FILTER_OPTS_ERROR
            if (   !strcmp(filter->filter->name, "format")     ||
                   !strcmp(filter->filter->name, "noformat")   ||
                   !strcmp(filter->filter->name, "frei0r")     ||
                   !strcmp(filter->filter->name, "frei0r_src") ||
                   !strcmp(filter->filter->name, "ocv")        ||
                   !strcmp(filter->filter->name, "pan")        ||
                   !strcmp(filter->filter->name, "pp")         ||
                   !strcmp(filter->filter->name, "aevalsrc")) {

            ......
            if (deprecated) {
                ret = AVERROR(EINVAL);
            } else {
                ret = process_options(filter, &options, copy);
            }
            av_freep(&copy);
            if (ret < 0)
                goto fail;
        } else
#endif
        {
			// 处理参数选项
            ret = process_options(filter, &options, args);
            if (ret < 0)
                goto fail;
        }
    }
    // 初始化参数选项
    ret = avfilter_init_dict(filter, &options);
    if (ret < 0)
        goto fail;
    if ((e = av_dict_get(options, "", NULL, AV_DICT_IGNORE_SUFFIX))) {
        ret = AVERROR_OPTION_NOT_FOUND;
        goto fail;
    }

fail:
    av_dict_free(&options);
    return ret;
}

其中，调用avfilter_init_dict()函数来初始化参数选项：

int avfilter_init_dict(AVFilterContext *ctx, AVDictionary **options)
{
    int ret = 0;

    ret = av_opt_set_dict(ctx, options);
    if (ret < 0) {
        return ret;
    }
    // 线程类型赋值
    if (ctx->filter->flags & AVFILTER_FLAG_SLICE_THREADS &&
        ctx->thread_type & ctx->graph->thread_type & AVFILTER_THREAD_SLICE &&
        ctx->graph->internal->thread_execute) {
        ctx->thread_type       = AVFILTER_THREAD_SLICE;
        ctx->internal->execute = ctx->graph->internal->thread_execute;
    } else {
        ctx->thread_type = 0;
    }
    if (ctx->filter->priv_class) {
        ret = av_opt_set_dict2(ctx->priv, options, AV_OPT_SEARCH_CHILDREN);
        if (ret < 0) {
            return ret;
        }
    }
    if (ctx->filter->init_opaque)
        ret = ctx->filter->init_opaque(ctx, NULL);
    else if (ctx->filter->init)
        ret = ctx->filter->init(ctx);
    else if (ctx->filter->init_dict)
        ret = ctx->filter->init_dict(ctx, options);

    if (ctx->enable_str) {
        ret = set_enable_expr(ctx, ctx->enable_str);
        if (ret < 0)
            return ret;
    }
    return ret;
}

 7、avfilter_insert_filter

avfilter_insert_filter()函数，在滤波链的两个滤波器之间插入新的滤波器，调用avfilter_link()函数执行真正的连接。代码如下：

int avfilter_insert_filter(AVFilterLink *link, AVFilterContext *filt,
                           unsigned filt_srcpad_idx, unsigned filt_dstpad_idx)
{
    int ret;
    unsigned dstpad_idx = link->dstpad - link->dst->input_pads;

    link->dst->inputs[dstpad_idx] = NULL;
	// 开始连接滤波器
    if ((ret = avfilter_link(filt, filt_dstpad_idx, link->dst, dstpad_idx)) < 0) {
        // 连接滤波器失败
        link->dst->inputs[dstpad_idx] = link;
        return ret;
    }
    link->dst                     = filt;
    link->dstpad                  = &filt->input_pads[filt_srcpad_idx];
    filt->inputs[filt_srcpad_idx] = link;
    // 保存相关的媒体格式信息
    if (link->outcfg.formats)
        ff_formats_changeref(&link->outcfg.formats,
                             &filt->outputs[filt_dstpad_idx]->outcfg.formats);
    if (link->outcfg.samplerates)
        ff_formats_changeref(&link->outcfg.samplerates,
                             &filt->outputs[filt_dstpad_idx]->outcfg.samplerates);
    if (link->outcfg.channel_layouts)
        ff_channel_layouts_changeref(&link->outcfg.channel_layouts,
                                     &filt->outputs[filt_dstpad_idx]->outcfg.channel_layouts);

    return 0;
}

内部调用avfilter_link()函数去连接滤波器，主要进行赋值操作、初始化帧队列：

int avfilter_link(AVFilterContext *src, unsigned srcpad,
                  AVFilterContext *dst, unsigned dstpad)
{
    AVFilterLink *link;
    av_assert0(src->graph);
    av_assert0(dst->graph);
    av_assert0(src->graph == dst->graph);

    if (src->nb_outputs <= srcpad || dst->nb_inputs <= dstpad ||
        src->outputs[srcpad]      || dst->inputs[dstpad])
        return AVERROR(EINVAL);
    if (src->output_pads[srcpad].type != dst->input_pads[dstpad].type) {
        return AVERROR(EINVAL);
    }
    link = av_mallocz(sizeof(*link));
    if (!link)
        return AVERROR(ENOMEM);

    src->outputs[srcpad] = dst->inputs[dstpad] = link;
	// 输入输出的AVFilterContext、pad赋值
    link->src     = src;
    link->dst     = dst;
    link->srcpad  = &src->output_pads[srcpad];
    link->dstpad  = &dst->input_pads[dstpad];
    link->type    = src->output_pads[srcpad].type;
    link->format  = -1;
	// 初始化帧队列
    ff_framequeue_init(&link->fifo, &src->graph->internal->frame_queues);

    return 0;
}

8、graph_config_formats

配置所有连接到滤波器图表的格式，包括四个步骤：寻找支持的格式、合并格式、确认格式、选择格式。具体如下：

static int graph_config_formats(AVFilterGraph *graph, AVClass *log_ctx)
{
    int ret;

    // 寻找支持的格式
    while ((ret = query_formats(graph, log_ctx)) == AVERROR(EAGAIN))
        av_log(graph, AV_LOG_DEBUG, "query_formats not finished\n");
    if (ret < 0)
        return ret;

    // 合并格式
    if ((ret = reduce_formats(graph)) < 0)
        return ret;

    // 音频：确认选中最佳的采样格式、采样率、声道布局
    swap_sample_fmts(graph);
    swap_samplerates(graph);
    swap_channel_layouts(graph);
    // 选择格式
    if ((ret = pick_formats(graph)) < 0)
        return ret;

    return 0;
}

其中，pick_formats()函数会遍历图表的所有滤波器，调用pick_format()函数进行选择格式。对于视频，选择最佳像素格式；对于音频，选择最佳采样格式。具体如下：

static int pick_format(AVFilterLink *link, AVFilterLink *ref)
{
    if (!link || !link->incfg.formats)
        return 0;

    if (link->type == AVMEDIA_TYPE_VIDEO) {
		// 视频：选择最佳的像素格式
        if(ref && ref->type == AVMEDIA_TYPE_VIDEO){
            int has_alpha= av_pix_fmt_desc_get(ref->format)->nb_components % 2 == 0;
            enum AVPixelFormat best= AV_PIX_FMT_NONE;
            int i;
            for (i = 0; i < link->incfg.formats->nb_formats; i++) {
                enum AVPixelFormat p = link->incfg.formats->formats[i];
                best= av_find_best_pix_fmt_of_2(best, p, ref->format, has_alpha, NULL);
            }
            link->incfg.formats->formats[0] = best;
        }
    } else if (link->type == AVMEDIA_TYPE_AUDIO) {
		// 音频：选择最佳的采样格式
        if(ref && ref->type == AVMEDIA_TYPE_AUDIO){
            enum AVSampleFormat best= AV_SAMPLE_FMT_NONE;
            int i;
            for (i = 0; i < link->incfg.formats->nb_formats; i++) {
                enum AVSampleFormat p = link->incfg.formats->formats[i];
                best = find_best_sample_fmt_of_2(best, p, ref->format);
            }
            link->incfg.formats->formats[0] = best;
        }
    }
    ......

    return 0;
}

 在选择最佳采样格式时，调用find_best_sample_fmt_of_2()函数，比较两种格式的分数，最终调用get_fmt_score()函数计算格式的得分：

static int get_fmt_score(enum AVSampleFormat dst_fmt, enum AVSampleFormat src_fmt)
{
    int score = 0;
    // 音频采样格式：无交错(平面存储)
    if (av_sample_fmt_is_planar(dst_fmt) != av_sample_fmt_is_planar(src_fmt))
        score ++;
    // 获取每个采样的字节数
    if (av_get_bytes_per_sample(dst_fmt) < av_get_bytes_per_sample(src_fmt)) {
        score += 100 * (av_get_bytes_per_sample(src_fmt) - av_get_bytes_per_sample(dst_fmt));
    }else
        score += 10  * (av_get_bytes_per_sample(dst_fmt) - av_get_bytes_per_sample(src_fmt));
    // 音频采样格式：交错存储，并且是32位或浮点数
    if (av_get_packed_sample_fmt(dst_fmt) == AV_SAMPLE_FMT_S32 &&
        av_get_packed_sample_fmt(src_fmt) == AV_SAMPLE_FMT_FLT)
        score += 20;
    if (av_get_packed_sample_fmt(dst_fmt) == AV_SAMPLE_FMT_FLT &&
        av_get_packed_sample_fmt(src_fmt) == AV_SAMPLE_FMT_S32)
        score += 2;

    return score;
}

9、avfilter_graph_free

avfilter_graph_free()函数，用于释放AVFilterGraph结构体：

void avfilter_graph_free(AVFilterGraph **graph)
{
    if (!*graph)
        return;
    // 释放滤波器
    while ((*graph)->nb_filters)
        avfilter_free((*graph)->filters[0]);
    // 释放滤波线程
    ff_graph_thread_free(*graph);
    av_freep(&(*graph)->sink_links);
    av_freep(&(*graph)->scale_sws_opts);
    av_freep(&(*graph)->aresample_swr_opts);
#if FF_API_LAVR_OPTS
    av_freep(&(*graph)->resample_lavr_opts);
#endif
    av_freep(&(*graph)->filters);
    av_freep(&(*graph)->internal);
    // 最终释放AVFilterGraph
    av_freep(graph);
}