【FFmpeg】Filter主要结构体及流程

Filter

Filter简介

视频特效处理功能：

• scale：视频/图像的缩放；
• overlay：视频/图片的叠加；
• crop：视频/图像的裁剪；
• trim：截取视频的片段；
• rotate：以任意⻆度旋转视频
• movie：加载第三方的视频；
• yadif：去隔行。

主要结构体：

• AVFilterGraph

  • 对filter系统的整体管理；主要用于统合这整个滤波过程

typedef struct AVFilterGraph {
    const AVClass *av_class;
    AVFilterContext **filters;
    unsigned nb_filters;
    ......
} AVFilterGraph;

• AVFilter

  • 滤波器的实现是通过AVFilter以及位于其下的结构体/函数来维护

  • 两个特殊的filter : buffer, buffersink

    • buffer：滤波器buffer代表filter graph中的源头，原始数据就往这个filter节点输入的。通过调用该滤波器提供的函数（如av_buffersrc_add_frame）可以把需要滤波的帧传输进入滤波过程。在创建该滤波器实例的时候需要提供一些关于所输入的帧的格式的必要参数（如：time_base、图像的宽高、图像像素格式等）。

    • buffersink：一个特殊的滤波器，滤波器buffersink代表filter graph中的输出节点，处理完成的数据从这个filter节点输出。通过调用滤波器提供的函数（如av_buffersink_get_frame）可以提供出被滤波过程过滤完成后的帧。

 /**
 * Filter definition. This defines the pads a filter contains, and all the
 * callback functions used to interact with the filter.
 */
    typedef struct AVFilter {
    const char *name;
    const char *description;
    const AVFilterPad *inputs;
    const AVFilterPad *outputs;
    const AVClass *priv_class;
    int flags;
    
    // private API
    ......
    } AVFilter;

• AVFilterContext

  • 一个滤波器实例，即使是同一个滤波器，在进行实际的滤波时，也会由于输入的参数不同而有不同的滤波效果，AVFilterContext就是在实际进行滤波时用于维护滤波相关信息的实体

/** An instance of a filter */
struct AVFilterContext {
    const AVClass *av_class;        ///< needed for av_log() and filters common options
    const AVFilter *filter;         ///< the AVFilter of which this is an instance
    char *name;                     ///< name of this filter instance
    AVFilterPad   *input_pads;      ///< array of input pads
    AVFilterLink **inputs;          ///< array of pointers to input links
    unsigned    nb_inputs;          ///< number of input pads
    AVFilterPad   *output_pads;     ///< array of output pads
    AVFilterLink **outputs;         ///< array of pointers to output links
    unsigned    nb_outputs;         ///< number of output pads
    void *priv;                     ///< private data for use by the filter
    struct AVFilterGraph *graph;    ///< filtergraph this filter belongs to
    ......
};

• AVFilterLink
  • 滤波器链，主要是用于链接相邻的两个AVFilterContext，为了实现一个滤波过程，可能会需要多个滤波器协同完成，即一个滤波器的输出可能会是另一个滤波器输入，AVFilterLink的作用是串联两个相邻的滤波器实例，形成两个滤波器之间的通道

/**
 * A link between two filters. This contains pointers to the source and
 * destination filters between which this link exists, and the indexes of
 * the pads involved. In addition, this link also contains the parameters
 * which have been negotiated and agreed upon between the filter, such as
 * image dimensions, format, etc.
 *
 * Applications must not normally access the link structure directly.
 * Use the buffersrc and buffersink API instead.
 * In the future, access to the header may be reserved for filters
 * implementation.
 */
struct AVFilterLink {
    AVFilterContext *src;       ///< source filter
    AVFilterPad *srcpad;        ///< output pad on the source filter
    AVFilterContext *dst;       ///< dest filter
    AVFilterPad *dstpad;        ///< input pad on the dest filter
    ......
}

• AVFilterPad

  • 滤波器的输入输出端口，一个滤波器可以有多个输入以及多个输出端口，相邻滤波器之间是通过AVFilterLink来串联的，而位于AVFilterLink两端的分别就是前一个滤波器的输出端口以及一个滤波器的输入端口

• AVFilterInOut

  • 过滤器链输入/输出的链接列表

/**
 * A linked-list of the inputs/outputs of the filter chain.
 *
 * This is mainly useful for avfilter_graph_parse() / avfilter_graph_parse2(),
 * where it is used to communicate open (unlinked) inputs and outputs from and
 * to the caller.
 * This struct specifies, per each not connected pad contained in the graph, the
 * filter context and the pad index required for establishing a link.
 */
typedef struct AVFilterInOut {
    /** unique name for this input/output in the list */
    char *name;
 
    /** filter context associated to this input/output */
    AVFilterContext *filter_ctx;
 
    /** index of the filt_ctx pad to use for linking */
    int pad_idx;
 
    /** next input/input in the list, NULL if this is the last */
    struct AVFilterInOut *next;
} AVFilterInOut;

主要函数：

//进行滤波器注册
avfilter_register_all();

// 获取FFmpeg中定义的filter，调用该方法前需要先调用avfilter_register_all();进行滤波器注册
AVFilter avfilter_get_by_name(const char name);

// 往源滤波器buffer中输入待处理的数据
int av_buffersrc_add_frame(AVFilterContext ctx, AVFrame frame);

// 从目的滤波器buffersink中获取处理完的数据
int av_buffersink_get_frame(AVFilterContext ctx, AVFrame frame);

// 创建一个滤波器图filter graph
AVFilterGraph *avfilter_graph_alloc(void);

// 创建一个滤波器实例AVFilterContext，并添加到AVFilterGraph中
int avfilter_graph_create_filter(AVFilterContext **filt_ctx, 
								const AVFilter *filt,const char name, 
								const char args, void *opaque,AVFilterGraph *graph_ctx);

// 连接两个滤波器节点
int avfilter_link(AVFilterContext *src, unsigned srcpad,AVFilterContext *dst, unsigned dstpad);

流程：

step1：注册过滤器

 avfilter_register_all();//注册

step2：创建统合整个滤波过程的滤波图结构体（AVFilterGraph）

AVFilterGraph* filter_graph = avfilter_graph_alloc();

step3：获取滤波过程需要的滤波器（AVFilter）

const AVFilter *buffersrc = avfilter_get_by_name("buffer");		//AVFilterGraph的输入源
const AVFilter *buffersink = avfilter_get_by_name("buffersink");//输出滤波器
const AVFilter *myfilter = avfilter_get_by_name("myfilter");	//处理业务的滤波器

step4：创建用于维护滤波相关信息的滤波器实例（AVFilterContext）

AVFilterContext *in_video_filter = NULL;
AVFilterContext *out_video_filter = NULL;
AVFilterContext *my_video_filter = NULL;

  char args[512];
    sprintf(args,
        "video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
        in_width, in_height, AV_PIX_FMT_YUV420P,
        1, 25, 1, 1);
avfilter_graph_create_filter(&in_video_filter, buffersrc, "in", args,NULL, filter_graph);
avfilter_graph_create_filter(&out_video_filter, buffersink, "out", NULL, NULL, filter_graph);
avfilter_graph_create_filter(&my_video_filter, myfilter, "myfilter",NULL, NULL, filter_graph);

step5：用AVFilterLink把相邻的两个滤波实例连接起来

avfilter_link(in_video_filter, 0, my_video_filter, 0);
avfilter_link(my_video_filter, 0, out_video_filter, 0);

step6：提交整个滤波图

avfilter_graph_config(filter_graph, NULL);

简单示例

/**
 * Copyright (c) 2022
 * Date: 2022年4月28日
  * Main Developer:
  * Developer:
 * Description: 给视频加上图片水印，如果是有α通道的透明png图片则可以显示为一个形状的图标
 * Refer: ffmpeg example filtering_video.c
 */

#include <io.h>
#include <process.h>
#include <stdio.h>
#include <stdlib.h>
extern "C" {
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavfilter/buffersink.h>
#include <libavfilter/buffersrc.h>
#include <libavutil/opt.h>
}

static AVFormatContext* fmt_ctx;
static AVCodecContext* dec_ctx;
//滤镜上下文 与 buffersrc 及 buffersink 对应
AVFilterContext* buffersink_ctx;
AVFilterContext* buffersrc_ctx;
AVFilterGraph* filter_graph;
static int video_stream_index = -1;

//打开输入文件
static int open_input_file(const char* filename)
{
  int ret;
  AVCodec* dec;

  if ((ret = avformat_open_input(&fmt_ctx, filename, NULL, NULL)) < 0) {
    av_log(NULL, AV_LOG_ERROR, "Cannot open input file\n");
    return ret;
  }

  if ((ret = avformat_find_stream_info(fmt_ctx, NULL)) < 0) {
    av_log(NULL, AV_LOG_ERROR, "Cannot find stream information\n");
    return ret;
  }

  //选择视频流
  ret = av_find_best_stream(fmt_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, &dec, 0);
  if (ret < 0) {
    av_log(NULL, AV_LOG_ERROR, "Cannot find a video stream in the input file\n");
    return ret;
  }
  video_stream_index = ret;

  /* create decoding context */
  dec_ctx = avcodec_alloc_context3(dec);
  if (!dec_ctx)
    return AVERROR(ENOMEM);
  avcodec_parameters_to_context(dec_ctx, fmt_ctx->streams[video_stream_index]->codecpar);

  //初始化解码器
  if ((ret = avcodec_open2(dec_ctx, dec, NULL)) < 0) {
    av_log(NULL, AV_LOG_ERROR, "Cannot open video decoder\n");
    return ret;
  }

  return 0;
}

//初始化filter
static int init_filters(const char* filters_descr)
{
  char args[512];
  int ret = 0;
  //滤镜输入缓冲区，解码器解码后的数据都会放到buffer中，是一个特殊的filter
  const AVFilter* buffersrc = avfilter_get_by_name("buffer");
  //滤镜输出缓冲区，滤镜处理完后输出的数据都会放在buffersink中，是一个特殊的filter
  const AVFilter* buffersink = avfilter_get_by_name("buffersink");
  AVFilterInOut* outputs = avfilter_inout_alloc();
  AVFilterInOut* inputs = avfilter_inout_alloc();
  AVRational time_base = fmt_ctx->streams[video_stream_index]->time_base;
  enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_YUV420P, AV_PIX_FMT_GRAY8, AV_PIX_FMT_NONE };

  //创建filter图，会包含本次使用到的所有过滤器
  filter_graph = avfilter_graph_alloc();
  if (!outputs || !inputs || !filter_graph) {
    ret = AVERROR(ENOMEM);
    goto end;
  }

  /* buffer video source: the decoded frames from the decoder will be inserted here. */
  snprintf(args, sizeof(args),
    "video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
    dec_ctx->width, dec_ctx->height, dec_ctx->pix_fmt,
    time_base.num, time_base.den,
    dec_ctx->sample_aspect_ratio.num, dec_ctx->sample_aspect_ratio.den);

  //创建过滤器实例并将其添加到现有graph中
  ret = avfilter_graph_create_filter(&buffersrc_ctx, buffersrc, "in",
    args, NULL, filter_graph);
  if (ret < 0) {
    av_log(NULL, AV_LOG_ERROR, "Cannot create buffer source\n");
    goto end;
  }

  /* 缓冲视频接收器: 终止过滤器链 */
  ret = avfilter_graph_create_filter(&buffersink_ctx, buffersink, "out",
    NULL, NULL, filter_graph);
  if (ret < 0) {
    av_log(NULL, AV_LOG_ERROR, "Cannot create buffer sink\n");
    goto end;
  }

  ret = av_opt_set_int_list(buffersink_ctx, "pix_fmts", pix_fmts,
    AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN);
  if (ret < 0) {
    av_log(NULL, AV_LOG_ERROR, "Cannot set output pixel format\n");
    goto end;
  }

  /*
   * Set the endpoints for the filter graph. The filter_graph will
   * be linked to the graph described by filters_descr.
   */

   /*
    * The buffer source output must be connected to the input pad of
    * the first filter described by filters_descr; since the first
    * filter input label is not specified, it is set to "in" by
    * default.
    */
  outputs->name = av_strdup("in");
  outputs->filter_ctx = buffersrc_ctx;
  outputs->pad_idx = 0;
  outputs->next = NULL;

  /*
   * The buffer sink input must be connected to the output pad of
   * the last filter described by filters_descr; since the last
   * filter output label is not specified, it is set to "out" by
   * default.
   */
  inputs->name = av_strdup("out");
  inputs->filter_ctx = buffersink_ctx;
  inputs->pad_idx = 0;
  inputs->next = NULL;

  //将由字符串描述的图形添加到图形中
  if ((ret = avfilter_graph_parse_ptr(filter_graph, filters_descr,
    &inputs, &outputs, NULL)) < 0)
    goto end;

  if ((ret = avfilter_graph_config(filter_graph, NULL)) < 0)
    goto end;

end:
  avfilter_inout_free(&inputs);
  avfilter_inout_free(&outputs);

  return ret;
}

//保存YUV数据
static int save_frame(AVFrame* filt_frame, FILE* out) {

  av_log(NULL, AV_LOG_DEBUG, "do_frame %d\n", filt_frame->format);
  if (filt_frame->format == AV_PIX_FMT_YUV420P) {
    av_log(NULL, AV_LOG_ERROR, "save 1 frame\n");
    for (int i = 0; i < filt_frame->height; i++) {
      fwrite(filt_frame->data[0] + filt_frame->linesize[0] * i, 1, filt_frame->width, out);
    }
    for (int i = 0; i < filt_frame->height / 2; i++) {
      fwrite(filt_frame->data[1] + filt_frame->linesize[1] * i, 1, filt_frame->width / 2, out);
    }
    for (int i = 0; i < filt_frame->height / 2; i++) {
      fwrite(filt_frame->data[2] + filt_frame->linesize[2] * i, 1, filt_frame->width / 2, out);
    }
  }
  fflush(out);

  return 0;
}


int main(int argc, char** argv)
{
  int ret;
  AVPacket packet;
  AVFrame* frame;
  AVFrame* filt_frame;
  FILE* out = NULL;

  //const char* filter_desc = "movie=qq.png[wm];[in][wm]overlay=5:5[out]"; //左上角绘制一个logo图片
  const char* filter_desc = "drawtext=:text=welcome:x=text_w:y=h-2*text_h:fontsize=30:fontcolor=red@0.7"; //左下角绘制文字
  //"drawbox=30:10:64:64:red";//x= 30,y=10,width=64,height=64,color=red,绘制一个红色正方形
  const char* filename = "E:/common/video_720_1.mp4";
  const char* outfile = "E:/common/video_720_1.yuv";

  av_log_set_level(AV_LOG_DEBUG);


  frame = av_frame_alloc();
  filt_frame = av_frame_alloc();
  if (!frame || !filt_frame) {
    perror("Could not allocate frame");
    exit(1);
  }
  out = fopen(outfile, "wb");
  if (!out) {
    av_log(NULL, AV_LOG_ERROR, "Failed to open yuv file!\n");
    exit(-1);
  }

  if ((ret = open_input_file(filename)) < 0)
    goto end;
  if ((ret = init_filters(filter_desc)) < 0)
    goto end;

  /* read all packets */
  while (1) {
    // 1 
    if ((ret = av_read_frame(fmt_ctx, &packet)) < 0)
      break;

    if (packet.stream_index == video_stream_index) {
      //2
      ret = avcodec_send_packet(dec_ctx, &packet);
      if (ret < 0) {
        av_log(NULL, AV_LOG_ERROR, "Error while sending a packet to the decoder\n");
        break;
      }

      while (ret >= 0) {
        //3
        ret = avcodec_receive_frame(dec_ctx, frame);
        if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
          break;
        }
        else if (ret < 0) {
          av_log(NULL, AV_LOG_ERROR, "Error while receiving a frame from the decoder\n");
          goto end;
        }

        frame->pts = frame->best_effort_timestamp;

        /*4 push the decoded frame into the filtergraph */
        if (av_buffersrc_add_frame_flags(buffersrc_ctx, frame, AV_BUFFERSRC_FLAG_KEEP_REF) < 0) {
          av_log(NULL, AV_LOG_ERROR, "Error while feeding the filtergraph\n");
          break;
        }

        /* pull filtered frames from the filtergraph */
        while (1) {
          //5
          ret = av_buffersink_get_frame(buffersink_ctx, filt_frame);
          if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
            break;
          if (ret < 0)
            goto end;
          save_frame(filt_frame, out);
          av_frame_unref(filt_frame);
        }
        av_frame_unref(frame);
      }
    }
    av_packet_unref(&packet);
  }
end:
  avfilter_graph_free(&filter_graph);
  avcodec_free_context(&dec_ctx);
  avformat_close_input(&fmt_ctx);
  av_frame_free(&frame);
  av_frame_free(&filt_frame);

  if (ret < 0 && ret != AVERROR_EOF) {
    //fprintf(stderr, "Error occurred: %s\n", av_strerror(ret));
    exit(1);
  }

  exit(0);
}

reference:

• http://ffmpeg.org/ffmpeg-filters.html#Filtering-Introduction
• https://blog.csdn.net/fuhanghang/article/details/124866293