FFmpeg 提取运动矢量表-extract_mvs

我发现用FFmpeg提取运动矢量比HM提取更加方便好用！

1. 参考
   （1）https://blog.csdn.net/ricky90/article/details/79436422
   （2）https://github.com/FFmpeg/FFmpeg/blob/release/4.1/doc/examples/extract_mvs.c

2.示例说明
示例来源于[1]，程序的流程如下所示。

FFmpeg提供了获取编码的运动矢量的方法。

• 打开解码器的时候设置参数：av_dict_set(&opts, “flags2”, “+export_mvs”, 0)。
• 使用av_frame_get_side_data(frame, AV_FRAME_DATA_MOTION_VECTORS)来获取解码frame中的运动矢量。

av_frame_get_side_data返回的数据类型为AVFrameSideData*，AVFrameSideData定义在libavutil/frame.h，如下所示。

/**
 * Structure to hold side data for an AVFrame.
 *
 * sizeof(AVFrameSideData) is not a part of the public ABI, so new fields may be added
 * to the end with a minor bump.
 */
typedef struct AVFrameSideData {
    enum AVFrameSideDataType type;
    uint8_t *data;
    int      size;
    AVDictionary *metadata;
    AVBufferRef *buf;
} AVFrameSideData;

• AVFrameSideDataType：表示数据的类型，用来存储运动矢量数据时，AVFrameSideDataType 为AV_FRAME_DATA_MOTION_VECTORS。
• data：指向数据buffer的指针，AVFrameSideDataType 为AV_FRAME_DATA_MOTION_VECTORS，data指向的地址存储的是AVMotionVector类型的数据。
• size：data指向的数据buffer的大小。

AVMotionVector是表示运动矢量的数据结构，定义在libavutil/motion_vector.h，如下所示。

typedef struct AVMotionVector {
    /**
     * Where the current macroblock comes from; negative value when it comes
     * from the past, positive value when it comes from the future.
     * XXX: set exact relative ref frame reference instead of a +/- 1 "direction".
     */
    int32_t source;
    /**
     * Width and height of the block.
     */
    uint8_t w, h;
    /**
     * Absolute source position. Can be outside the frame area.
     */
    int16_t src_x, src_y;
    /**
     * Absolute destination position. Can be outside the frame area.
     */
    int16_t dst_x, dst_y;
    /**
     * Extra flag information.
     * Currently unused.
     */
    uint64_t flags;
    /**
     * Motion vector
     * src_x = dst_x + motion_x / motion_scale
     * src_y = dst_y + motion_y / motion_scale
     */
    int32_t motion_x, motion_y;
    uint16_t motion_scale;
} AVMotionVector;

参数说明：

• int32_t source：当前像素参考的帧来源，负值表示时参考过去的帧，正值表示参考未来的帧。
• uint8_t w, h：block的宽和高。
• int16_t src_x, src_y：源的绝对位置。可能在frame之外。
• int16_t dst_x, dst_y：目的的绝对位置。可能在frame之外。
• uint16_t motion_scale：运动矢量的像素精度，4则表示1/4像素。
• int32_t motion_x, motion_y: 运动的矢量。满足下面的等式。

src_x = dst_x + motion_x / motion_scale
src_y = dst_y + motion_y / motion_scale

3. 示例代码

/*
 * Copyright (c) 2012 Stefano Sabatini
 * Copyright (c) 2014 Clément Bœsch
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

#include <libavutil/motion_vector.h>
#include <libavformat/avformat.h>

static AVFormatContext *fmt_ctx = NULL;
static AVCodecContext *video_dec_ctx = NULL;
static AVStream *video_stream = NULL;
static const char *src_filename = NULL;

static int video_stream_idx = -1;
static AVFrame *frame = NULL;
static int video_frame_count = 0;

static int decode_packet(const AVPacket *pkt)
{
    int ret = avcodec_send_packet(video_dec_ctx, pkt);
    if (ret < 0) {
        fprintf(stderr, "Error while sending a packet to the decoder: %s\n", av_err2str(ret));
        return ret;
    }

    while (ret >= 0)  {
        ret = avcodec_receive_frame(video_dec_ctx, frame);
        if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
            break;
        } else if (ret < 0) {
            fprintf(stderr, "Error while receiving a frame from the decoder: %s\n", av_err2str(ret));
            return ret;
        }

        if (ret >= 0) {
            int i;
            AVFrameSideData *sd;

            video_frame_count++;
            sd = av_frame_get_side_data(frame, AV_FRAME_DATA_MOTION_VECTORS);
            if (sd) {
                const AVMotionVector *mvs = (const AVMotionVector *)sd->data;
                for (i = 0; i < sd->size / sizeof(*mvs); i++) {
                    const AVMotionVector *mv = &mvs[i];
                    printf("%d,%2d,%2d,%2d,%4d,%4d,%4d,%4d,%d,0x%"PRIx64"\n",
                        video_frame_count, mv->source,
                        mv->w, mv->h, mv->src_x, mv->src_y,
                        mv->dst_x, mv->dst_y, mv->motion_scale, mv->flags);
                }
            }
            av_frame_unref(frame);
        }
    }

    return 0;
}

static int open_codec_context(AVFormatContext *fmt_ctx, enum AVMediaType type)
{
    int ret;
    AVStream *st;
    AVCodecContext *dec_ctx = NULL;
    AVCodec *dec = NULL;
    AVDictionary *opts = NULL;

    ret = av_find_best_stream(fmt_ctx, type, -1, -1, &dec, 0);
    if (ret < 0) {
        fprintf(stderr, "Could not find %s stream in input file '%s'\n",
                av_get_media_type_string(type), src_filename);
        return ret;
    } else {
        int stream_idx = ret;
        st = fmt_ctx->streams[stream_idx];

        dec_ctx = avcodec_alloc_context3(dec);
        if (!dec_ctx) {
            fprintf(stderr, "Failed to allocate codec\n");
            return AVERROR(EINVAL);
        }

        ret = avcodec_parameters_to_context(dec_ctx, st->codecpar);
        if (ret < 0) {
            fprintf(stderr, "Failed to copy codec parameters to codec context\n");
            return ret;
        }

        /* Init the video decoder */
        av_dict_set(&opts, "flags2", "+export_mvs", 0);
        if ((ret = avcodec_open2(dec_ctx, dec, &opts)) < 0) {
            fprintf(stderr, "Failed to open %s codec\n",
                    av_get_media_type_string(type));
            return ret;
        }

        video_stream_idx = stream_idx;
        video_stream = fmt_ctx->streams[video_stream_idx];
        video_dec_ctx = dec_ctx;
    }

    return 0;
}

int main(int argc, char **argv)
{
    int ret = 0;
    AVPacket pkt = { 0 };

    if (argc != 2) {
        fprintf(stderr, "Usage: %s <video>\n", argv[0]);
        exit(1);
    }
    src_filename = argv[1];

    if (avformat_open_input(&fmt_ctx, src_filename, NULL, NULL) < 0) {
        fprintf(stderr, "Could not open source file %s\n", src_filename);
        exit(1);
    }

    if (avformat_find_stream_info(fmt_ctx, NULL) < 0) {
        fprintf(stderr, "Could not find stream information\n");
        exit(1);
    }

    open_codec_context(fmt_ctx, AVMEDIA_TYPE_VIDEO);

    av_dump_format(fmt_ctx, 0, src_filename, 0);

    if (!video_stream) {
        fprintf(stderr, "Could not find video stream in the input, aborting\n");
        ret = 1;
        goto end;
    }

    frame = av_frame_alloc();
    if (!frame) {
        fprintf(stderr, "Could not allocate frame\n");
        ret = AVERROR(ENOMEM);
        goto end;
    }

    printf("framenum,source,blockw,blockh,srcx,srcy,dstx,dsty,motion_scale,flags\n");

    /* read frames from the file */
    while (av_read_frame(fmt_ctx, &pkt) >= 0) {
        if (pkt.stream_index == video_stream_idx)
            ret = decode_packet(&pkt);
        av_packet_unref(&pkt);
        if (ret < 0)
            break;
    }

    /* flush cached frames */
    decode_packet(NULL);

end:
    avcodec_free_context(&video_dec_ctx);
    avformat_close_input(&fmt_ctx);
    av_frame_free(&frame);
    return ret < 0;
}

部分结果如下所示。

framenum,source,blockw,blockh,srcx,srcy,dstx,dsty,motion_scale,flags
2,-1,16,16,   8,   8,   8,   8,4,0x0
2, 1,16,16,   8,   8,   8,   8,4,0x0
2,-1,16,16,  24,   8,  24,   8,4,0x0
2, 1,16,16,  24,   8,  24,   8,4,0x0
2,-1,16,16,  40,   8,  40,   8,4,0x0
2, 1,16,16,  40,   8,  40,   8,4,0x0
2,-1,16,16,  56,   8,  56,   8,4,0x0
2, 1,16,16,  56,   8,  56,   8,4,0x0
2,-1,16,16,  72,   8,  72,   8,4,0x0
2, 1,16,16,  72,   8,  72,   8,4,0x0
2,-1,16,16,  88,   8,  88,   8,4,0x0
2, 1,16,16,  88,   8,  88,   8,4,0x0
2,-1,16,16, 104,   8, 104,   8,4,0x0
2, 1,16,16, 104,   8, 104,   8,4,0x0
2,-1,16,16, 120,   8, 120,   8,4,0x0
2, 1,16,16, 120,   8, 120,   8,4,0x0
2,-1,16,16, 136,   8, 136,   8,4,0x0