ffmpeg实现画中画

本篇博客相比上一篇《 ffmpeg滤镜学习一，movie+overlay滤镜实现视频加水印、画中画》更深入一些，本次的实现，可以控制子画面出现的时间段、子画面播放时间等，这篇文章主要参考了大师兄悟空公众号下的文章《使用 FFmpeg 实现画中画效果(一)》，下面看一下具体实现：

首先提出5个问题：

1. 子画面展示位置？

2. 子画面从主画面的哪个时间点开始播放？

3. 子画面从子画面的哪个时间点开始播放？

4. 子画面是按照时间段显示还是一直显示？

5. 如果子画面和主画面不等长怎么办？

要解决这5个问题，主要使用overlay滤镜，如下：

liupeng@bj-navigate-liupeng:~$ ffmpeg -h filter=overlay
ffmpeg version N-95635-gcae7f66 Copyright (c) 2000-2019 the FFmpeg developers
  built with gcc 7 (Ubuntu 7.4.0-1ubuntu1~18.04.1)
  configuration: --prefix=/home/liupeng/ffmpeg_build --pkg-config-flags=--static --extra-cflags=-I/home/liupeng/ffmpeg_build/include --extra-ldflags=-L/home/liupeng/ffmpeg_build/lib --extra-libs='-lpthread -lm' --bindir=/home/liupeng/bin --enable-gpl --enable-libass --enable-libfdk-aac --enable-libfreetype --enable-libmp3lame --enable-libopus --enable-libvorbis --enable-libvpx --enable-libx264 --enable-libx265 --enable-nonfree
  libavutil      56. 35.101 / 56. 35.101
  libavcodec     58. 60.100 / 58. 60.100
  libavformat    58. 34.101 / 58. 34.101
  libavdevice    58.  9.100 / 58.  9.100
  libavfilter     7. 66.100 /  7. 66.100
  libswscale      5.  6.100 /  5.  6.100
  libswresample   3.  6.100 /  3.  6.100
  libpostproc    55.  6.100 / 55.  6.100
Filter overlay
  Overlay a video source on top of the input.
    slice threading supported
    Inputs:
       #0: main (video)
       #1: overlay (video)
    Outputs:
       #0: default (video)
overlay AVOptions:
  x                 <string>     ..FV...... set the x expression (default "0")
  y                 <string>     ..FV...... set the y expression (default "0")
  eof_action        <int>        ..FV...... Action to take when encountering EOF from secondary input  (from 0 to 2) (default repeat)
     repeat          0            ..FV...... Repeat the previous frame.
     endall          1            ..FV...... End both streams.
     pass            2            ..FV...... Pass through the main input.
  eval              <int>        ..FV...... specify when to evaluate expressions (from 0 to 1) (default frame)
     init            0            ..FV...... eval expressions once during initialization
     frame           1            ..FV...... eval expressions per-frame
  shortest          <boolean>    ..FV...... force termination when the shortest input terminates (default false)
  format            <int>        ..FV...... set output format (from 0 to 5) (default yuv420)
     yuv420          0            ..FV......
     yuv422          1            ..FV......
     yuv444          2            ..FV......
     rgb             3            ..FV......
     gbrp            4            ..FV......
     auto            5            ..FV......
  repeatlast        <boolean>    ..FV...... repeat overlay of the last overlay frame (default true)
  alpha             <int>        ..FV...... alpha format (from 0 to 1) (default straight)
     straight        0            ..FV......
     premultiplied   1            ..FV......

framesync AVOptions:
  eof_action        <int>        ..FV...... Action to take when encountering EOF from secondary input  (from 0 to 2) (default repeat)
     repeat          0            ..FV...... Repeat the previous frame.
     endall          1            ..FV...... End both streams.
     pass            2            ..FV...... Pass through the main input.
  shortest          <boolean>    ..FV...... force termination when the shortest input terminates (default false)
  repeatlast        <boolean>    ..FV...... extend last frame of secondary streams beyond EOF (default true)

This filter has support for timeline through the 'enable' option.

首先通过x、y参数可以解决子画面显示位置的问题。

shortest参数可以解决主画面、子画面时间不等的问题。

enable参数可以解决2、4两个问题，第三个问题需要使用一个新的滤镜setpts，主画面与子画面的视频偏移可以通过setpts滤镜设置，如下：

liupeng@bj-navigate-liupeng:~$ ffmpeg -h filter=setpts
ffmpeg version N-95635-gcae7f66 Copyright (c) 2000-2019 the FFmpeg developers
  built with gcc 7 (Ubuntu 7.4.0-1ubuntu1~18.04.1)
  configuration: --prefix=/home/liupeng/ffmpeg_build --pkg-config-flags=--static --extra-cflags=-I/home/liupeng/ffmpeg_build/include --extra-ldflags=-L/home/liupeng/ffmpeg_build/lib --extra-libs='-lpthread -lm' --bindir=/home/liupeng/bin --enable-gpl --enable-libass --enable-libfdk-aac --enable-libfreetype --enable-libmp3lame --enable-libopus --enable-libvorbis --enable-libvpx --enable-libx264 --enable-libx265 --enable-nonfree
  libavutil      56. 35.101 / 56. 35.101
  libavcodec     58. 60.100 / 58. 60.100
  libavformat    58. 34.101 / 58. 34.101
  libavdevice    58.  9.100 / 58.  9.100
  libavfilter     7. 66.100 /  7. 66.100
  libswscale      5.  6.100 /  5.  6.100
  libswresample   3.  6.100 /  3.  6.100
  libpostproc    55.  6.100 / 55.  6.100
Filter setpts
  Set PTS for the output video frame.
    Inputs:
       #0: default (video)
    Outputs:
       #0: default (video)
setpts AVOptions:
  expr              <string>     ..FVA..... Expression determining the frame timestamp (default "PTS")

设置画布：

const char *filter_descr = "movie=out1.mp4[in2];[in2]setpts=PTS[out2];[0:v][out2]overlay=x=20:y=120:enable='between(t,2,15)':shortest=1";

还是贴一下代码吧：

/*
 * 实现对现有视频增加水印，可以是图片、也可以是视频，若为视频，类似画中画
 */
#include "myffmpeg/util.h"
extern "C"
{
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/opt.h>
#include <libavfilter/buffersink.h>
#include <libavfilter/buffersrc.h>
    int open_input_file(AVFormatContext *fmt, AVCodecContext **codecctx, AVCodec *codec, const char *filename, int index)
    {
        int ret = 0;
        char msg[500];
        *codecctx = avcodec_alloc_context3(codec);
        ret = avcodec_parameters_to_context(*codecctx, fmt->streams[index]->codecpar);
        if (ret < 0)
        {
            sprintf(msg, "avcodec_parameters_to_context error,ret:%d\n", ret);
            lp_log(msg);
            return -1;
        }

        // open 解码器
        ret = avcodec_open2(*codecctx, codec, NULL);
        if (ret < 0)
        {
            sprintf(msg, "avcodec_open2 error,ret:%d\n", ret);
            lp_log(msg);
            return -2;
        }
        printf("pix:%d\n", (*codecctx)->pix_fmt);
        return ret;
    }

    int init_filter(AVFilterContext **buffersrc_ctx, AVFilterContext **buffersink_ctx, AVFilterGraph **filter_graph, AVStream *stream, AVCodecContext *codecctx, const char *filter_desc)
    {
        int ret = -1;
        char args[512];
        char msg[500];
        const AVFilter *buffersrc = avfilter_get_by_name("buffer");
        const AVFilter *buffersink = avfilter_get_by_name("buffersink");

        AVFilterInOut *input = avfilter_inout_alloc();
        AVFilterInOut *output = avfilter_inout_alloc();

        AVRational time_base = stream->time_base;
        enum AVPixelFormat pix_fmts[] = {AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE};

        if (!output || !input || !filter_graph)
        {
            ret = -1;
            sprintf(msg, "avfilter_graph_alloc/avfilter_inout_alloc error,ret:%d\n", ret);
            lp_log(msg);
            goto end;
        }
        snprintf(args, sizeof(args), "video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d", codecctx->width, codecctx->height, codecctx->pix_fmt, stream->time_base.num, stream->time_base.den, codecctx->sample_aspect_ratio.num, codecctx->sample_aspect_ratio.den);
        ret = avfilter_graph_create_filter(buffersrc_ctx, buffersrc, "in", args, NULL, *filter_graph);
        if (ret < 0)
        {
            sprintf(msg, "avfilter_graph_create_filter buffersrc error,ret:%d\n", ret);
            lp_log(msg);
            goto end;
        }

        ret = avfilter_graph_create_filter(buffersink_ctx, buffersink, "out", NULL, NULL, *filter_graph);
        if (ret < 0)
        {
            sprintf(msg, "avfilter_graph_create_filter buffersink error,ret:%d\n", ret);
            lp_log(msg);
            goto end;
        }
        ret = av_opt_set_int_list(*buffersink_ctx, "pix_fmts", pix_fmts, AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN);
        if (ret < 0)
        {
            sprintf(msg, "av_opt_set_int_list error,ret:%d\n", ret);
            lp_log(msg);
            goto end;
        }
        /*
     * The buffer source output must be connected to the input pad of
     * the first filter described by filters_descr; since the first
     * filter input label is not specified, it is set to "in" by
     * default.
     */
        output->name = av_strdup("in");
        output->filter_ctx = *buffersrc_ctx;
        output->pad_idx = 0;
        output->next = NULL;

        /*
     * The buffer sink input must be connected to the output pad of
     * the last filter described by filters_descr; since the last
     * filter output label is not specified, it is set to "out" by
     * default.
     */
        input->name = av_strdup("out");
        input->filter_ctx = *buffersink_ctx;
        input->pad_idx = 0;
        input->next = NULL;

        if ((ret = avfilter_graph_parse_ptr(*filter_graph, filter_desc, &input, &output, NULL)) < 0)
        {
            sprintf(msg, "avfilter_graph_parse_ptr error,ret:%d\n", ret);
            lp_log(msg);
            goto end;
        }

        if ((ret = avfilter_graph_config(*filter_graph, NULL)) < 0)
        {
            sprintf(msg, "avfilter_graph_config error,ret:%d\n", ret);
            lp_log(msg);
            goto end;
        }
    end:
        avfilter_inout_free(&input);
        avfilter_inout_free(&output);
        return ret;
    }

    int my_filter(const char *name)
    {
        int ret;
        char msg[500];
        // const char *filter_descr = "movie=my_logo.png[wm];[in][wm]overlay=10:10[out]";
        // const char *filter_descr = "scale=640:360,transpose=cclock";
        const char *filter_descr = "movie=out1.mp4[in2];[in2]setpts=PTS[out2];[in][out2]overlay=x=20:y=120:enable='between(t,2,15)':shortest=1";
        AVFormatContext *pFormatCtx = NULL;
        AVCodecContext *pCodecCtx;
        AVFilterContext *buffersink_ctx;
        AVFilterContext *buffersrc_ctx;
        AVFilterGraph *filter_graph;
        AVCodec *codec;
        int video_stream_index = -1;

        AVPacket packet;
        AVFrame *pFrame;
        AVFrame *pFrame_out;
        filter_graph = avfilter_graph_alloc();
        FILE *fp_yuv = fopen("test.yuv", "wb+");
        ret = avformat_open_input(&pFormatCtx, name, NULL, NULL);
        if (ret < 0)
        {
            sprintf(msg, "avformat_open_input error,ret:%d\n", ret);
            lp_log(msg);
            ret = -1;
            goto end;
        }

        ret = avformat_find_stream_info(pFormatCtx, NULL);
        if (ret < 0)
        {
            sprintf(msg, "avformat_find_stream_info error,ret:%d\n", ret);
            lp_log(msg);
            ret = -2;
            goto end;
        }

        ret = av_find_best_stream(pFormatCtx, AVMEDIA_TYPE_VIDEO, -1, -1, &codec, 0);
        if (ret < 0)
        {
            sprintf(msg, "av_find_best_stream error,ret:%d\n", ret);
            lp_log(msg);
            ret = -3;
            goto end;
        }
        // 获取到视频流索引
        video_stream_index = ret;

        av_dump_format(pFormatCtx, 0, name, 0);
        if ((ret = open_input_file(pFormatCtx, &pCodecCtx, codec, name, video_stream_index)) < 0)
        {
            ret = -4;
            sprintf(msg, "open_input_file error,ret:%d\n", ret);
            lp_log(msg);
            goto end;
        }

        if ((ret = init_filter(&buffersrc_ctx, &buffersink_ctx, &filter_graph, pFormatCtx->streams[video_stream_index], pCodecCtx, filter_descr)) < 0)
        {
            ret = -5;
            sprintf(msg, "init_filter error,ret:%d\n", ret);
            lp_log(msg);
            goto end;
        }
        pFrame = av_frame_alloc();
        pFrame_out = av_frame_alloc();
        while (1)
        {
            if ((ret = av_read_frame(pFormatCtx, &packet)) < 0)
                break;

            if (packet.stream_index == video_stream_index)
            {
                ret = avcodec_send_packet(pCodecCtx, &packet);
                if (ret < 0)
                {
                    sprintf(msg, "avcodec_send_packet error,ret:%d\n", ret);
                    lp_log(msg);
                    break;
                }

                while (ret >= 0)
                {
                    ret = avcodec_receive_frame(pCodecCtx, pFrame);
                    if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
                    {
                        break;
                    }
                    else if (ret < 0)
                    {
                        sprintf(msg, "avcodec_receive_frame error,ret:%d\n", ret);
                        lp_log(msg);
                        goto end;
                    }

                    pFrame->pts = pFrame->best_effort_timestamp;

                    /* push the decoded frame into the filtergraph */
                    ret = av_buffersrc_add_frame_flags(buffersrc_ctx, pFrame, AV_BUFFERSRC_FLAG_KEEP_REF);
                    if (ret < 0)
                    {
                        sprintf(msg, "av_buffersrc_add_frame_flags error,ret:%d\n", ret);
                        lp_log(msg);
                        break;
                    }

                    /* pull filtered frames from the filtergraph */
                    while (1)
                    {
                        ret = av_buffersink_get_frame(buffersink_ctx, pFrame_out);
                        if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
                            break;
                        if (ret < 0)
                        {
                            ret = -6;
                            goto end;
                        }
                        if (pFrame_out->format == AV_PIX_FMT_YUV420P)
                        {
                            //Y, U, V
                            for (int i = 0; i < pFrame_out->height; i++)
                            {
                                fwrite(pFrame_out->data[0] + pFrame_out->linesize[0] * i, 1, pFrame_out->width, fp_yuv);
                            }
                            for (int i = 0; i < pFrame_out->height / 2; i++)
                            {
                                fwrite(pFrame_out->data[1] + pFrame_out->linesize[1] * i, 1, pFrame_out->width / 2, fp_yuv);
                            }
                            for (int i = 0; i < pFrame_out->height / 2; i++)
                            {
                                fwrite(pFrame_out->data[2] + pFrame_out->linesize[2] * i, 1, pFrame_out->width / 2, fp_yuv);
                            }
                        }
                        av_frame_unref(pFrame_out);
                    }
                    av_frame_unref(pFrame);
                }
            }
            av_packet_unref(&packet);
        }
    end:
        avcodec_free_context(&pCodecCtx);
        fclose(fp_yuv);
    }
}

觉着这篇文章对自己有益的土豪朋友可以扫描屏幕下方二维码金额随意，感谢大家支持，增加写作动力。