ffmpeg + SDL 实现简单的视频播放器

前言

本文借鉴了雷神的遗作， 同时对一些已经舍弃的API更新， 加上自己的理解， 解释了若干初学者也是本人刚开始的一些疑问， 实现了一个丐版的视频播放器， 算是对以前学习的复习总结， 本文的demo仅仅能实现视频的播放， 其他播放器功能会逐渐完善。

流程图

对流程图的若干解释：
1， 整体分为三部分， 最左边是为了实现解码FFMPEG)， 最右边的是为了实现图形化显示（SDL)， 中间是sws_scale部分
2. sws_scale 部分是为了实现ffmpeg 解码后的YUV数据的转换， 主要包括采样格式，分辨率， 或者需要进行filter 操作。

代码实现

#include <iostream>
#include <SDL2/SDL.h>

#ifdef __cplusplus
extern "C" {
#include <libavcodec/avcodec.h>
#include <libavfilter/avfilter.h>
#include <libavformat/avformat.h>
#include <libavutil/avutil.h>
#include <libavutil/log.h>
#include <libavdevice/avdevice.h>
#include <libswresample/swresample.h>
#include <libswscale/swscale.h>
#include <libavutil/imgutils.h>
}
#else
#include <libavcodec/avcodec.h>
#include <libavfilter/avfilter.h>
#include <libavformat/avformat.h>
#include <libavutil/avutil.h>
#include <libavutil/log.h>
#include <libavdevice/avdevice.h>
#include <libswresample/swresample.h>
#include <libswscale/swscale.h>
#include <libavutil/imgutils.h>
#endif

static int threadQuit = 0;

#define VEDIO_FILE1  "D:\\ZLJ\\stuff\\Vedio\\111.mkv"
#define VEDIO_FILE  "D:\\ZLJ\\stuff\\Vedio\\111.mkv"

#define EVENT_FREASH  (SDL_USEREVENT + 1)
static bool pause = 0;
#undef main

int threadFunc(void *argv) {
    SDL_Event event;
    threadQuit = 0;
    while (!threadQuit) {
        if (!pause) {
            event.type = EVENT_FREASH;
            SDL_PushEvent(&event);
            SDL_Delay(50);
        }
    }
    return 0;
}
using namespace std;
int main()
{
    AVFormatContext         *pFormatCtx = nullptr;
    AVCodecContext          *pCodecCtx  = nullptr;
    AVCodec                 *pCodec     = nullptr;
    AVCodecParameters       *pCodecParm = nullptr;
    AVPacket                *pPacket    = nullptr;
    AVFrame                 *pFrame     = nullptr;
    AVFrame                 *pFrameYuv  = nullptr;
    SwsContext              *pSwsCtx    = nullptr;
    
    SDL_Window              *window     = nullptr;
    SDL_Renderer            *render     = nullptr;
    SDL_Texture             *texture    = nullptr;
    SDL_Thread              *thread     = nullptr;
    SDL_Event               event;
    
    int                     rst = 0, vedioIndex = -1;
    int                     windowW, windowH;
    unsigned char           *outBuffer  = nullptr;
    
    av_log_set_level(AV_LOG_INFO);
    SDL_Init(SDL_INIT_VIDEO);

    rst = avformat_open_input(&pFormatCtx, VEDIO_FILE, nullptr, nullptr);
    if (rst < 0) {
        av_log(nullptr, AV_LOG_ERROR, "open input failed\n");
        goto _EIXT2;
    }
    
    rst = avformat_find_stream_info(pFormatCtx, nullptr);
    if (rst < 0) {
        av_log(nullptr, AV_LOG_ERROR, "find stream info failed\n");
        goto _EIXT2;
    }
    
    rst = vedioIndex = av_find_best_stream(pFormatCtx,  AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0);
    if (rst < 0) {
        av_log(nullptr, AV_LOG_ERROR, "find best stream failed\n");
        goto _EIXT2;
    }
    else {
        av_log(nullptr, AV_LOG_INFO, "the vedioIndex = %d\n", vedioIndex);
    }
    
    pCodecParm = pFormatCtx->streams[vedioIndex]->codecpar;
    pCodec = avcodec_find_decoder(pCodecParm->codec_id);
    if (pCodec == nullptr) {
        av_log(nullptr, AV_LOG_ERROR, "find decoder failed \n");
        goto _EIXT2;
    }
    else {
        av_log(nullptr, AV_LOG_INFO, "width = %d, height = %d\n", pCodecParm->width, pCodecParm->height);
    }
    pCodecCtx = avcodec_alloc_context3(pCodec);
    if (pCodecCtx == nullptr) {
        av_log(nullptr, AV_LOG_ERROR, "alloc avcodec failed \n");
        goto _EIXT2;
    }
    rst = avcodec_parameters_to_context(pCodecCtx, pCodecParm);
    if (rst < 0) {
        av_log(nullptr, AV_LOG_ERROR, "avcodec_parameters_to_context failed\n");
        goto _EIXT3;
    }
    else {
        av_log(nullptr, AV_LOG_INFO, "from codecCtx width = %d, height = %d\n", pCodecCtx->width, pCodecCtx->height);
    }
    
    rst = avcodec_open2(pCodecCtx, pCodec, nullptr);
    if (rst < 0) {
        av_log(nullptr, AV_LOG_ERROR, "avcodec_open2 failed\n");
        goto _EIXT4;
    }
    
    pPacket = av_packet_alloc();
    if (pPacket == nullptr) {
        av_log(nullptr, AV_LOG_ERROR, "av_packet_alloc failed\n");
        goto _EIXT5;
    }
    av_init_packet(pPacket);
    pFrame      = av_frame_alloc();
    pFrameYuv   = av_frame_alloc();
    
    windowH     = pCodecParm->height;
    windowW     = pCodecParm->width;
    windowH     = (windowH >> 4) << 4;
    windowW     = (windowW >> 4) << 4;

    outBuffer = static_cast<unsigned char *>(av_malloc(static_cast<size_t>(av_image_get_buffer_size(AV_PIX_FMT_YUV420P, windowW, windowH, 1))));
    if (outBuffer == nullptr) {
        av_log(nullptr, AV_LOG_ERROR, "malloc out buff failed\n");
        goto _EIXT5;
    }
    else {
        av_log(nullptr, AV_LOG_INFO, "malloc out buff for yuv frame successfully\n");
    }
    rst = av_image_fill_arrays(pFrameYuv->data, pFrameYuv->linesize, outBuffer, AV_PIX_FMT_YUV420P, windowW, windowH, 1);
    if (rst < 0) {
        av_log(nullptr, AV_LOG_ERROR, "fill array failed\n");
        goto _EIXT6;
    }
    else {
        av_log(nullptr, AV_LOG_INFO, "av_image_fill_array successfully\n");
    }
    pSwsCtx = sws_alloc_context();
    rst = sws_init_context(pSwsCtx, nullptr, nullptr);
    if (rst < 0) {
        av_log(nullptr, AV_LOG_ERROR, "init sws_context failed \n");
        goto _EIXT6;
    }
    pSwsCtx = sws_getContext(pCodecParm->width, pCodecParm->height,  pCodecCtx->pix_fmt,
                             windowW, windowH, AV_PIX_FMT_YUV420P, SWS_BICUBIC, nullptr, nullptr, nullptr);
    if (pSwsCtx == nullptr) {
        av_log(nullptr, AV_LOG_ERROR, "get sws context failed\n");
        goto _EXIT7;
    }
    else {
        av_log(nullptr, AV_LOG_INFO, "sws_getContext implement successfully\n");
    }
    
    cout << windowW << "    " << windowH << endl;
    window = SDL_CreateWindow("longjiang", 100, 100, windowW, windowH,
                              SDL_WINDOW_SHOWN|SDL_WINDOW_OPENGL|SDL_WINDOW_RESIZABLE);
    render = SDL_CreateRenderer(window, -1, 0);
    texture= SDL_CreateTexture(render, SDL_PIXELFORMAT_IYUV, SDL_TEXTUREACCESS_TARGET, windowW, windowH);
    SDL_SetRenderDrawColor(render, 255, 100, 100, 255);
    
    thread = SDL_CreateThread(threadFunc, nullptr, nullptr);
    while (1) {
        SDL_WaitEvent(&event);
        if (event.type == EVENT_FREASH ) {
            while (1) {
                if (av_read_frame(pFormatCtx, pPacket) < 0) {
                    threadQuit = 1;
                    break;
                }
                if (pPacket->stream_index == vedioIndex) {
                    rst = avcodec_send_packet(pCodecCtx, pPacket);
                    if (rst < 0) {
                        av_log(nullptr, AV_LOG_ERROR, "send packet faild\n");
                        continue;
                    } else {
                        break;
                    }
                }
            }
            while (!threadQuit && rst >= 0) {
                rst = avcodec_receive_frame(pCodecCtx, pFrame);
                if (rst == AVERROR(EAGAIN) || rst == AVERROR_EOF)
                    break;
                if (!threadQuit && rst == 0) {
                    rst = sws_scale(pSwsCtx, pFrame->data, pFrame->linesize, 0, pFrame->height, pFrameYuv->data, pFrameYuv->linesize);
                    if (rst < 0) {
                        av_log(nullptr, AV_LOG_ERROR, "sws_scale implement failed\n");
                        break;
                    }
                    SDL_RenderClear(render);
                    SDL_UpdateYUVTexture(texture, nullptr,
                                         pFrameYuv->data[0], pFrameYuv->linesize[0],
                            pFrameYuv->data[1], pFrameYuv->linesize[1],
                            pFrameYuv->data[2], pFrameYuv->linesize[2]);
                    SDL_RenderCopy(render, texture, nullptr, nullptr);
                    SDL_RenderPresent(render);
                }
            }
            
            av_packet_unref(pPacket);
        } else if (threadQuit == 1) {
            break;
        } else if (event.type == SDL_QUIT) {
            threadQuit = 1;
            break;
        } else if (event.type == SDL_WINDOWEVENT) {
            SDL_GetWindowSize(window, &windowW, &windowH);
        } else if (event.type == SDL_KEYUP) {
            if (event.key.keysym.sym == SDLK_SPACE) {
                pause = !pause;
            }
        }
    }

    //flush decode
    {
        rst = avcodec_send_packet(pCodecCtx, nullptr);
        av_log(nullptr, AV_LOG_ERROR, "hahahahhahahahahahahhahah rst = %d\n", rst);
        while (!threadQuit && rst >= 0) {
            rst = avcodec_receive_frame(pCodecCtx, pFrame);
            if (rst == AVERROR(EAGAIN) || rst == AVERROR_EOF)
                break;
            if (!threadQuit && rst == 0) {
                rst = sws_scale(pSwsCtx, pFrame->data, pFrame->linesize, 0, pFrame->height, pFrameYuv->data, pFrameYuv->linesize);
                if (rst < 0) {
                    av_log(nullptr, AV_LOG_ERROR, "sws_scale implement failed\n");
                    break;
                }
                SDL_RenderClear(render);
                SDL_UpdateYUVTexture(texture, nullptr,
                                     pFrameYuv->data[0], pFrameYuv->linesize[0],
                        pFrameYuv->data[1], pFrameYuv->linesize[1],
                        pFrameYuv->data[2], pFrameYuv->linesize[2]);
                SDL_RenderCopy(render, texture, nullptr, nullptr);
                SDL_RenderPresent(render);
                av_log(nullptr, AV_LOG_ERROR, "hahahahhahahahahahahhahah\n");
            }
        }
    }
    
    avcodec_send_packet(pCodecCtx, pPacket);
    rst = avcodec_receive_frame(pCodecCtx, pFrame);
    threadQuit = 1;
    
    SDL_DestroyWindow(window);
    SDL_DestroyTexture(texture);
    SDL_DestroyRenderer(render);
    SDL_Quit();
_EXIT7:
    sws_freeContext(pSwsCtx);
_EIXT6:
    av_free(outBuffer);
_EIXT5:
    avcodec_close(pCodecCtx);
_EIXT4:
    avcodec_parameters_free(&pCodecParm);
_EIXT3:
    avcodec_free_context(&pCodecCtx);
_EIXT2:
    avformat_close_input(&pFormatCtx);
    
    cout << "Hello World!" << endl;
    return 0;
}

若干问题的深度解析

为什么要用sws_scale

大家要想明白为什么需要sws_scale操作， 别一看一过就没了， 因为我们没法确定我们解析出来的YUV数据的具体格式， 但是ffmpeg 和 SDL对采样格式的表示是不一样的， 为了方便显示， 就把我们解析出来的YUV数据， 统一为某一种固定的格式
PS：我看网上有一种说法是： 因为frame->data[] 这里面的数据比YUV的每一个平面的数据都大， 后面多出来的都是无用的数据。所以需要裁剪掉了。对于这种说我本人持保留意见， 在receive的时候已经知道了分辨率和YUV格式， ffmpeg为什么会这么笨呢，由于源码实在庞大， 我还没有看的太明白， 希望有大佬不吝赐教！

为什么需要flush decoder

流处理结束的时候需要flush（洗刷） codec。因为codec可能在内部缓冲多个frame或packet，出于性能或其他必要的情况（如考虑B帧的情况, B帧是双向预测帧， 为了相邻帧能正确编码， 需要把B帧缓存了）。
官方解释如下：
http://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
At the beginning of decoding or encoding, the codec might accept multiple input frames/packets without returning a frame, until its internal buffers are filled. This situation is handled transparently if you follow the steps outlined above.

In theory, sending input can result in EAGAIN - this should happen only if not all output was received. You can use this to structure alternative decode or encode loops other than the one suggested above. For example, you could try sending new input on each iteration, and try to receive output if that returns EAGAIN.

End of stream situations. These require “flushing” (aka draining) the codec, as the codec might buffer multiple frames or packets internally for performance or out of necessity (consider B-frames). This is handled as follows:

Instead of valid input, send NULL to the avcodec_send_packet() (decoding) or avcodec_send_frame() (encoding) functions. This will enter draining mode.
Call avcodec_receive_frame() (decoding) or avcodec_receive_packet() (encoding) in a loop until AVERROR_EOF is returned. The functions will not return AVERROR(EAGAIN), unless you forgot to enter draining mode.
Before decoding can be resumed again, the codec has to be reset with avcodec_flush_buffers().
Using the API as outlined above is highly recommended. But it is also possible to call functions outside of this rigid schema. For example, you can call avcodec_send_packet() repeatedly without calling avcodec_receive_frame(). In this case, avcodec_send_packet() will succeed until the codec’s internal buffer has been filled up (which is typically of size 1 per output frame, after initial input), and then reject input with AVERROR(EAGAIN). Once it starts rejecting input, you have no choice but to read at least some output.

Not all codecs will follow a rigid and predictable dataflow; the only guarantee is that an AVERROR(EAGAIN) return value on a send/receive call on one end implies that a receive/send call on the other end will succeed, or at least will not fail with AVERROR(EAGAIN). In general, no codec will permit unlimited buffering of input or output.

怎么flush decoder

关于这个问题， 困扰了好久， 蓦然回首才发现竟在灯火阑珊处。

请大家精读 parameter (in)，大家一定好好好读几遍comment。
大致意思是每一个packet可能包含了好几个frame, 就需要多次receive frame, 如果传入的pacakt为nullptr， 则是告诉decoder要结束了， 来flush了。 第一次flush会返回成功

 * Supply raw packet data as input to a decoder.
 *  * Internally, this call will copy relevant AVCodecContext fields, which can
 * influence decoding per-packet, and apply them when the packet is actually
 * decoded. (For example AVCodecContext.skip_frame, which might direct the
 * decoder to drop the frame contained by the packet sent with this function.)
 *  * @warning The input buffer, avpkt->data must be AV_INPUT_BUFFER_PADDING_SIZE
 *          larger than the actual read bytes because some optimized bitstream
 *          readers read 32 or 64 bits at once and could read over the end.
 *  * @warning Do not mix this API with the legacy API (like avcodec_decode_video2())
 *          on the same AVCodecContext. It will return unexpected results now
 *          or in future libavcodec versions.
 *  * @note The AVCodecContext MUST have been opened with @ref avcodec_open2()
 *       before packets may be fed to the decoder.
 *  * @param avctx codec context
 * @param[in] avpkt The input AVPacket. Usually, this will be a single video
 *                  frame, or several complete audio frames.
 *                  Ownership of the packet remains with the caller, and the
 *                  decoder will not write to the packet. The decoder may create
 *                  a reference to the packet data (or copy it if the packet is
 *                  not reference-counted).
 *                  Unlike with older APIs, the packet is always fully consumed,
 *                  and if it contains multiple frames (e.g. some audio codecs),
 *                  will require you to call avcodec_receive_frame() multiple
 *                  times afterwards before you can send a new packet.
 *                  It can be NULL (or an AVPacket with data set to NULL and
 *                  size set to 0); in this case, it is considered a flush
 *                  packet, which signals the end of the stream. Sending the
 *                  first flush packet will return success. Subsequent ones are
 *                  unnecessary and will return AVERROR_EOF. If the decoder
 *                  still has frames buffered, it will return them after sending
 *                  a flush packet.
 *  * @return 0 on success, otherwise negative error code:
 *      AVERROR(EAGAIN):   input is not accepted in the current state - user
 *                         must read output with avcodec_receive_frame() (once
 *                         all output is read, the packet should be resent, and
 *                         the call will not fail with EAGAIN).
 *      AVERROR_EOF:       the decoder has been flushed, and no new packets can
 *                         be sent to it (also returned if more than 1 flush
 *                         packet is sent)
 *      AVERROR(EINVAL):   codec not opened, it is an encoder, or requires flush
 *      AVERROR(ENOMEM):   failed to add packet to internal queue, or similar
 *      other errors: legitimate decoding errors
 */
int avcodec_send_packet(AVCodecContext *avctx, const AVPacket *avpkt);

总结：

• 调用avcodec_send_*()传入的AVFrame或AVPacket指针设置为NULL。 这将开启draining mode（排水模式）。
• 反复地调用avcodec_receive_*()直到返回AVERROR_EOF的错误，这个方法这个时候不会返回AVERROR(EAGAIN)的错误，除非你忘记了开启draining mode。
• codec可以重新开启，但是需要先调用 avcodec_flush_buffers()来重置codec。

重点说明

• 编码或者解码刚开始的时候，codec可能接收了多个输入的frame或packet后还没有输出数据，直到内部的buffer被填充满。上面的使用流程可以处理这种情况。
• 理论上调用avcodec_send_()的时候可能会发生AVERROR(EAGAIN)的错误，这只应该在有输出数据没有被接收的情况，你可以依赖这个机制来实现区别于上面建议流程的处理方式，比如反复地调用avcodec_send_()，出现AVERROR(EAGAIN)错误的时候再去调用avcodec_receive_*()。
• 并不是所有的codec都遵循一个严格、可预测的数据处理流程，唯一可以保证的是 “调用avcodec_send_()/avcodec_receive_()返回AVERROR(EAGAIN)的时候去avcodec_receive_()/avcodec_send_()会成功，否则不应该返回AVERROR(EAGAIN)的错误。”一般来说，任何codec都不允许无限制地缓冲输入或者输出。

写在最后的话

十分感谢雷神给我辈留下的不朽财富， 极其鄙视那些只知道原文照搬的copyer, 很多弃用的API还在用， 没有一点新意。 但是雷神确实难以超越。

本博客参考学习了：https://www.jianshu.com/p/d77718947e21
http://www.ffmpeg.org/doxygen/4.1/index.html
不朽的雷神