FFmpeg在libavfilter模块提供简单与复杂的音视频滤镜,所有滤波器由AVFilterGraph滤波器图表连接起来。简单滤镜为一对一输出,复杂滤镜为多对一输出。重要的结构体包括:AVFilterGraph、AVFilterLink、AVFilterContext、AVFilter。支持在滤波器图表指定位置插入AVFilter滤波器,然后由AVFilterLink把滤波器连接起来。
1、AVFilterGraph
AVFilterGraph为滤波器图表结构体,保存滤波器上下文、滤波器数量、线程类型、线程数量、回调函数等,代码位于libavfilter/avfilter.h头文件中:
typedef struct AVFilterGraph {
const AVClass *av_class;
// 滤波器上下文
AVFilterContext **filters;
// 滤波器数量
unsigned nb_filters;
char *scale_sws_opts;
int thread_type;
int nb_threads;
AVFilterGraphInternal *internal;
void *opaque;
// 用户设置的回调函数
avfilter_execute_func *execute;
char *aresample_swr_opts;
AVFilterLink **sink_links;
int sink_links_count;
unsigned disable_auto_convert;
} AVFilterGraph;
2、AVFilterLink
AVFilterLink为滤波器的连接器,负责把两个滤波器连接起来,内部包含指向源滤波器和目标滤波器的指针。需要注意的是,应用程序不能直接访问AVFilterLink,使用buffersrc和buffersink的API代替。代码同样位于avfilter.h头文件中:
struct AVFilterLink {
// 源滤波器上下文
AVFilterContext *src;
AVFilterPad *srcpad;
// 目标滤波器上下文
AVFilterContext *dst;
AVFilterPad *dstpad;
// 媒体类型
enum AVMediaType type;
// 视频参数
int w;
int h;
AVRational sample_aspect_ratio;
// 音频参数
uint64_t channel_layout;
int sample_rate;
int format;
// pts的时间基
AVRational time_base;
AVFilterFormatsConfig incfg;
AVFilterFormatsConfig outcfg;
// 连接器的初始化状态
enum {
AVLINK_UNINIT = 0, // 没初始化
AVLINK_STARTINIT, // 已初始化,没完成
AVLINK_INIT // 已完成初始化
} init_state;
struct AVFilterGraph *graph;
// 当前pts时间戳
int64_t current_pts;
int64_t current_pts_us;
int age_index;
AVRational frame_rate;
AVFrame *partial_buf;
int partial_buf_size;
int min_samples;
int max_samples;
int channels;
int64_t frame_count_in, frame_count_out;
void *frame_pool;
int frame_wanted_out;
AVBufferRef *hw_frames_ctx;
#ifndef FF_INTERNAL_FIELDS
// 内部数据结构(保留)
char reserved[0xF000];
#else /* FF_INTERNAL_FIELDS */
// 滤波器的帧队列
FFFrameQueue fifo;
frame_blocked_in;
int status_in;
int64_t status_in_pts;
int status_out;
#endif /* FF_INTERNAL_FIELDS */
};
3、AVFilterContext
AVFilterContext为滤波器上下文的结构体,包含滤波器实例、名字、输入Pad和Link、输出Pad和Link,还有ready字段表示滤波器准备就绪且带有优先级。具体代码如下:
struct AVFilterContext {
const AVClass *av_class;
// 滤波器实例
const AVFilter *filter;
// 滤波器名字
char *name;
AVFilterPad *input_pads;
AVFilterLink **inputs;
unsigned nb_inputs;
AVFilterPad *output_pads;
AVFilterLink **outputs;
unsigned nb_outputs;
void *priv;
struct AVFilterGraph *graph;
int thread_type;
AVFilterInternal *internal;
struct AVFilterCommand *command_queue;
char *enable_str;
void *enable;
double *var_values;
int is_disabled;
AVBufferRef *hw_device_ctx;
int nb_threads;
// 滤波器准备就绪,带优先级
unsigned ready;
int extra_hw_frames;
};
4、AVFilter
AVFilter为滤波器的结构体,包含滤波器名字与描述、输入输出的AVFilterPad、滤波器操作的函数指针。具体代码如下:
typedef struct AVFilter {
// 滤波器名字
const char *name;
// 滤波器描述
const char *description;
const AVFilterPad *inputs;
const AVFilterPad *outputs;
const AVClass *priv_class;
int flags;
int priv_size;
int flags_internal;
// 预初始化函数指针
int (*preinit)(AVFilterContext *ctx);
// 初始化函数指针
int (*init)(AVFilterContext *ctx);
// 带字典参数的初始化函数指针
int (*init_dict)(AVFilterContext *ctx, AVDictionary **options);
// 反初始化函数指针
void (*uninit)(AVFilterContext *ctx);
// 查询滤波器支持的输入输出格式
int (*query_formats)(AVFilterContext *);
// 处理命令函数指针
int (*process_command)(AVFilterContext *, const char *cmd,
const char *arg, char *res, int res_len, int flags);
// 带透传指针的初始化函数指针
int (*init_opaque)(AVFilterContext *ctx, void *opaque);
// 滤波器激活函数指针
int (*activate)(AVFilterContext *ctx);
} AVFilter;
5、avfilter_graph_create_filter
负责创建并添加滤波器实例,并且添加到AVFilterGraph图表。代码如下:
int avfilter_graph_create_filter(AVFilterContext **filt_ctx, const AVFilter *filt,
const char *name, const char *args, void *opaque,
AVFilterGraph *graph_ctx)
{
int ret;
// 分配滤波器上下文
*filt_ctx = avfilter_graph_alloc_filter(graph_ctx, filt, name);
if (!*filt_ctx)
return AVERROR(ENOMEM);
// 初始化滤波器
ret = avfilter_init_str(*filt_ctx, args);
if (ret < 0)
goto fail;
return 0;
fail:
if (*filt_ctx)
avfilter_free(*filt_ctx);
*filt_ctx = NULL;
return ret;
}
调用avfilter_graph_alloc_filter()函数来分配滤波器上下文,调用ff_graph_thread_init()来创建滤波线程,调用ff_filter_alloc()来真正分配滤波器上下文。代码如下:
AVFilterContext *avfilter_graph_alloc_filter(AVFilterGraph *graph,
const AVFilter *filter,
const char *name)
{
AVFilterContext **filters, *s;
// 创建滤波器线程
if (graph->thread_type && !graph->internal->thread_execute) {
if (graph->execute) {
graph->internal->thread_execute = graph->execute;
} else {
int ret = ff_graph_thread_init(graph);
if (ret < 0) {
return NULL;
}
}
}
// 分配滤波器上下文
s = ff_filter_alloc(filter, name);
if (!s)
return NULL;
filters = av_realloc(graph->filters, sizeof(*filters) * (graph->nb_filters + 1));
if (!filters) {
avfilter_free(s);
return NULL;
}
graph->filters = filters;
graph->filters[graph->nb_filters++] = s;
s->graph = graph;
return s;
}
6、avfilter_init_str
avfilter_init_str()函数,用于初始化滤波器,包括兼容旧版命令语法与新版命令语法。代码位于avfilter.c中:
int avfilter_init_str(AVFilterContext *filter, const char *args)
{
AVDictionary *options = NULL;
AVDictionaryEntry *e;
int ret = 0;
if (args && *args) {
if (!filter->filter->priv_class) {
return AVERROR(EINVAL);
}
// 兼容旧版语法
#if FF_API_OLD_FILTER_OPTS_ERROR
if ( !strcmp(filter->filter->name, "format") ||
!strcmp(filter->filter->name, "noformat") ||
!strcmp(filter->filter->name, "frei0r") ||
!strcmp(filter->filter->name, "frei0r_src") ||
!strcmp(filter->filter->name, "ocv") ||
!strcmp(filter->filter->name, "pan") ||
!strcmp(filter->filter->name, "pp") ||
!strcmp(filter->filter->name, "aevalsrc")) {
......
if (deprecated) {
ret = AVERROR(EINVAL);
} else {
ret = process_options(filter, &options, copy);
}
av_freep(©);
if (ret < 0)
goto fail;
} else
#endif
{
// 处理参数选项
ret = process_options(filter, &options, args);
if (ret < 0)
goto fail;
}
}
// 初始化参数选项
ret = avfilter_init_dict(filter, &options);
if (ret < 0)
goto fail;
if ((e = av_dict_get(options, "", NULL, AV_DICT_IGNORE_SUFFIX))) {
ret = AVERROR_OPTION_NOT_FOUND;
goto fail;
}
fail:
av_dict_free(&options);
return ret;
}
其中,调用avfilter_init_dict()函数来初始化参数选项:
int avfilter_init_dict(AVFilterContext *ctx, AVDictionary **options)
{
int ret = 0;
ret = av_opt_set_dict(ctx, options);
if (ret < 0) {
return ret;
}
// 线程类型赋值
if (ctx->filter->flags & AVFILTER_FLAG_SLICE_THREADS &&
ctx->thread_type & ctx->graph->thread_type & AVFILTER_THREAD_SLICE &&
ctx->graph->internal->thread_execute) {
ctx->thread_type = AVFILTER_THREAD_SLICE;
ctx->internal->execute = ctx->graph->internal->thread_execute;
} else {
ctx->thread_type = 0;
}
if (ctx->filter->priv_class) {
ret = av_opt_set_dict2(ctx->priv, options, AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
return ret;
}
}
if (ctx->filter->init_opaque)
ret = ctx->filter->init_opaque(ctx, NULL);
else if (ctx->filter->init)
ret = ctx->filter->init(ctx);
else if (ctx->filter->init_dict)
ret = ctx->filter->init_dict(ctx, options);
if (ctx->enable_str) {
ret = set_enable_expr(ctx, ctx->enable_str);
if (ret < 0)
return ret;
}
return ret;
}
7、avfilter_insert_filter
avfilter_insert_filter()函数,在滤波链的两个滤波器之间插入新的滤波器,调用avfilter_link()函数执行真正的连接。代码如下:
int avfilter_insert_filter(AVFilterLink *link, AVFilterContext *filt,
unsigned filt_srcpad_idx, unsigned filt_dstpad_idx)
{
int ret;
unsigned dstpad_idx = link->dstpad - link->dst->input_pads;
link->dst->inputs[dstpad_idx] = NULL;
// 开始连接滤波器
if ((ret = avfilter_link(filt, filt_dstpad_idx, link->dst, dstpad_idx)) < 0) {
// 连接滤波器失败
link->dst->inputs[dstpad_idx] = link;
return ret;
}
link->dst = filt;
link->dstpad = &filt->input_pads[filt_srcpad_idx];
filt->inputs[filt_srcpad_idx] = link;
// 保存相关的媒体格式信息
if (link->outcfg.formats)
ff_formats_changeref(&link->outcfg.formats,
&filt->outputs[filt_dstpad_idx]->outcfg.formats);
if (link->outcfg.samplerates)
ff_formats_changeref(&link->outcfg.samplerates,
&filt->outputs[filt_dstpad_idx]->outcfg.samplerates);
if (link->outcfg.channel_layouts)
ff_channel_layouts_changeref(&link->outcfg.channel_layouts,
&filt->outputs[filt_dstpad_idx]->outcfg.channel_layouts);
return 0;
}
内部调用avfilter_link()函数去连接滤波器,主要进行赋值操作、初始化帧队列:
int avfilter_link(AVFilterContext *src, unsigned srcpad,
AVFilterContext *dst, unsigned dstpad)
{
AVFilterLink *link;
av_assert0(src->graph);
av_assert0(dst->graph);
av_assert0(src->graph == dst->graph);
if (src->nb_outputs <= srcpad || dst->nb_inputs <= dstpad ||
src->outputs[srcpad] || dst->inputs[dstpad])
return AVERROR(EINVAL);
if (src->output_pads[srcpad].type != dst->input_pads[dstpad].type) {
return AVERROR(EINVAL);
}
link = av_mallocz(sizeof(*link));
if (!link)
return AVERROR(ENOMEM);
src->outputs[srcpad] = dst->inputs[dstpad] = link;
// 输入输出的AVFilterContext、pad赋值
link->src = src;
link->dst = dst;
link->srcpad = &src->output_pads[srcpad];
link->dstpad = &dst->input_pads[dstpad];
link->type = src->output_pads[srcpad].type;
link->format = -1;
// 初始化帧队列
ff_framequeue_init(&link->fifo, &src->graph->internal->frame_queues);
return 0;
}
8、graph_config_formats
配置所有连接到滤波器图表的格式,包括四个步骤:寻找支持的格式、合并格式、确认格式、选择格式。具体如下:
static int graph_config_formats(AVFilterGraph *graph, AVClass *log_ctx)
{
int ret;
// 寻找支持的格式
while ((ret = query_formats(graph, log_ctx)) == AVERROR(EAGAIN))
av_log(graph, AV_LOG_DEBUG, "query_formats not finished\n");
if (ret < 0)
return ret;
// 合并格式
if ((ret = reduce_formats(graph)) < 0)
return ret;
// 音频:确认选中最佳的采样格式、采样率、声道布局
swap_sample_fmts(graph);
swap_samplerates(graph);
swap_channel_layouts(graph);
// 选择格式
if ((ret = pick_formats(graph)) < 0)
return ret;
return 0;
}
其中,pick_formats()函数会遍历图表的所有滤波器,调用pick_format()函数进行选择格式。对于视频,选择最佳像素格式;对于音频,选择最佳采样格式。具体如下:
static int pick_format(AVFilterLink *link, AVFilterLink *ref)
{
if (!link || !link->incfg.formats)
return 0;
if (link->type == AVMEDIA_TYPE_VIDEO) {
// 视频:选择最佳的像素格式
if(ref && ref->type == AVMEDIA_TYPE_VIDEO){
int has_alpha= av_pix_fmt_desc_get(ref->format)->nb_components % 2 == 0;
enum AVPixelFormat best= AV_PIX_FMT_NONE;
int i;
for (i = 0; i < link->incfg.formats->nb_formats; i++) {
enum AVPixelFormat p = link->incfg.formats->formats[i];
best= av_find_best_pix_fmt_of_2(best, p, ref->format, has_alpha, NULL);
}
link->incfg.formats->formats[0] = best;
}
} else if (link->type == AVMEDIA_TYPE_AUDIO) {
// 音频:选择最佳的采样格式
if(ref && ref->type == AVMEDIA_TYPE_AUDIO){
enum AVSampleFormat best= AV_SAMPLE_FMT_NONE;
int i;
for (i = 0; i < link->incfg.formats->nb_formats; i++) {
enum AVSampleFormat p = link->incfg.formats->formats[i];
best = find_best_sample_fmt_of_2(best, p, ref->format);
}
link->incfg.formats->formats[0] = best;
}
}
......
return 0;
}
在选择最佳采样格式时,调用find_best_sample_fmt_of_2()函数,比较两种格式的分数,最终调用get_fmt_score()函数计算格式的得分:
static int get_fmt_score(enum AVSampleFormat dst_fmt, enum AVSampleFormat src_fmt)
{
int score = 0;
// 音频采样格式:无交错(平面存储)
if (av_sample_fmt_is_planar(dst_fmt) != av_sample_fmt_is_planar(src_fmt))
score ++;
// 获取每个采样的字节数
if (av_get_bytes_per_sample(dst_fmt) < av_get_bytes_per_sample(src_fmt)) {
score += 100 * (av_get_bytes_per_sample(src_fmt) - av_get_bytes_per_sample(dst_fmt));
}else
score += 10 * (av_get_bytes_per_sample(dst_fmt) - av_get_bytes_per_sample(src_fmt));
// 音频采样格式:交错存储,并且是32位或浮点数
if (av_get_packed_sample_fmt(dst_fmt) == AV_SAMPLE_FMT_S32 &&
av_get_packed_sample_fmt(src_fmt) == AV_SAMPLE_FMT_FLT)
score += 20;
if (av_get_packed_sample_fmt(dst_fmt) == AV_SAMPLE_FMT_FLT &&
av_get_packed_sample_fmt(src_fmt) == AV_SAMPLE_FMT_S32)
score += 2;
return score;
}
9、avfilter_graph_free
avfilter_graph_free()函数,用于释放AVFilterGraph结构体:
void avfilter_graph_free(AVFilterGraph **graph)
{
if (!*graph)
return;
// 释放滤波器
while ((*graph)->nb_filters)
avfilter_free((*graph)->filters[0]);
// 释放滤波线程
ff_graph_thread_free(*graph);
av_freep(&(*graph)->sink_links);
av_freep(&(*graph)->scale_sws_opts);
av_freep(&(*graph)->aresample_swr_opts);
#if FF_API_LAVR_OPTS
av_freep(&(*graph)->resample_lavr_opts);
#endif
av_freep(&(*graph)->filters);
av_freep(&(*graph)->internal);
// 最终释放AVFilterGraph
av_freep(graph);
}