FFMPEG SDK 开发介绍

1.简介:
   ffmpeg是一套可以用来记录、转换数字音频、视频，并能将其转化为流的开源计算机程序。
使用ffmpeg能够完成如下功能:parse,demux,decode,filter(preprocessing),encode,mux,stream和player等.

2.下载和编译:

   下载地址:  http://ffmpeg.org/download.html

    编译:
      1)windows平台static library/shared library,编译工具:mingw-gcc或者在linux平台下交叉编译(推荐)
      2)linux平台static library/shared library, 编译工具:gcc

    模块:
       libavcodec    -编码解码器
       libavdevice   - 输入输出设备的支持
       libavfilter   - 视音频滤镜支持
       libavformat   - 视音频等格式的解析
       libavutil    - 工具库
       libpostproc   - 后期效果处理
       libswscale    -图像颜色、尺寸转换
    
3.SDK介绍和开发(基于ffmpeg 0.8 sdk)
   ffmpeg每部分功能都采用plugin的方式，使用统一的接口调用，这样就能够非常方便的使用和扩展。
   plugin分为几种:muxer,demuxer,protocol,hwaccel,encoder,decoder,parser,bitstream,filter,...
   因此在使用SDK的时候第一步就是注册plugin
    
   avcodec_register_all()  : 注册hwaccel,encoder,decoder,parser,bitstream
   av_register_all()      : 注册 muxer,demuxer,protocol
   avfilter_register_all() : 注册 滤镜filter
    
   下面根据不同的应用场景，给出主要的代码示例(仅是代码片断，不一定能编译通过):
    
   1)如何获取媒体文件的信息(Parser):
    // 参考V3代码:interface IFileDecoder, media/impl/filedecoderimpl.cpp
    
    {
       av_register_all();
       AVFormatContext * pFormatCtx = NULL;
       int err = 0;
       const char *fileName = "c:\\test.mp4";
       err = av_open_input_file(&pFormatCtx, fileName,NULL, 0, NULL);
       if(err != 0)
       {
           // break ;
       }
       err = av_find_stream_info(pFormatCtx);
       if(err < 0)
       {
           // break ;
       }
       for(uint32_t i = 0; i <pFormatCtx->nb_streams; i ++)
       {
           // stream 结构数据
           AVStream *pStream = pFormatCtx->streams[i];
           // 帧率信息
           AVRational frameRate = pStream->r_frame_rate;
           // 时间单位比率
           AVRational timeBase = pStream->time_base;
           // stream duration
           int64_t duration = pStream->duration;
           
           // 获取Codec数据结构
           AVCodecContext *pCodecCtx = pStream->codec;
           AVMediaType codecType = pCodecCtx->codec_type;
           
           CodecID codecId = pCodecCtx->codec_id;
           
           
           if(codecType == AVMEDIA_TYPE_VIDEO)
           {
               // 获取Video基本信息
               int width = pCodecCtx->width;
               int height = pCodecCtx->height;
               PixelFormat pixelFormat = pCodecCtx->pix_fmt;
           }
           else if(codecType == AVMEDIA_TYPE_AUDIO)
           {
               // 获取Audio基本信息
               int channels = pCodecCtx->channels;
               int sample_rate = pCodecCtx->sample_rate;
               AVSampleFormat sampleFmt =pCodecCtx->sample_fmt;
           }
       }
       // 释放
       if(pFormatCtx != NULL)
       {
           av_close_input_file(pFormatCtx);
           pFormatCtx = NULL;
       }    
    }
    
   2)读取sample数据(Read raw sample不解码)
    // 参考V3代码:interface IFileDecoder, media/impl/filedecoderimpl.cpp

    {
       // 参考Parser代码
       // av_register_all();
       // AVFormatContext * pFormatCtx = NULL;
       // err = av_open_input_file(&pFormatCtx, fileName,NULL, 0, NULL);
    
       AVPacket packet;
       av_init_packet(&packet);
       int ret = av_read_frame(pFormatCtx, &packet);
       if(ret >= 0)
       {
           int streamIndex = packet.stream_index;
           AVStream *pStream =pFormatCtx->streams[streamIndex];
           AVCodecContext *pCodecCtx = pStream->codec;
           // 计算timestamp
    
           // 转换时间到1/1000000秒
           AVRational time_base;
           time_base.num = 1;
           time_base.den = 1000000;
           
         //25.0    1/25,  29.97   1001/30000
         
           // 获取 dts/pts
           const int64_t dts = av_rescale_q(packet.dts,pStream->time_base, time_base);
           const int64_t pts = av_rescale_q(packet.pts,pStream->time_base, time_base);
           uint8_t *data = packet.data;
           int size = packet.size;
           bool isKey = ((packet.flags & AV_PKT_FLAG_KEY) ==AV_PKT_FLAG_KEY);   
       }
       av_free_packet(&packet);       
    }
    
   3)解码sample(Video ES=>YUV/RGB, Audio ES=>PCM)
    // 参考V3代码:interface IVideoDecoder/IAudioDecoder,media/impl/videodecoderimpl.cpp/audiodecoderimpl.cpp
    {
       // 参考Parser,Read raw sample代码
       
       // AVMediaType codecType =pCodecCtx->codec_type;
       AVMediaType codecType = AVMEDIA_TYPE_VIDEO;
       // CodecId codecId = pCodecCtx->codec_id;
       CodecId codecId = CODEC_ID_H264;
       
       // 通过Codec ID查找解码器
       AVCodec *pCodec = avcodec_find_decoder(codecId);
       // 分配codec关联结构
       AVCodecContext *pCodecCtx = avcodec_alloc_context();

       // 设置一些必要的信息
       pCodecCtx->codec_type = AVMEDIA_TYPE_VIDEO /AVMEDIA_TYPE_AUDIO;
       pCodecCtx->codec_id  = codecId;

       if(pCodec->capabilities &CODEC_CAP_TRUNCATED)
           pCodecCtx->flags |= CODEC_FLAG_TRUNCATED;

       // 在open codec时要加锁，否则多个codec同时打开时时会出现错误
       gMutexFFmpeg.lock();       
       // 打开Codec
       avcodec_open(pCodecCtx,pCodec);       
       gMutexFFmpeg.unlock();
       
       if(codecType == AVMEDIA_TYPE_VIDEO)
       {
           AVFrame *pSrcFrame = avcodec_alloc_frame();
           AVFrame *pDstFrame = avcodec_alloc_frame();
           
           // 因为内存的原因，所以需要多分配一些数据, FF_INPUT_BUFFER_PADDING_SIZE
           uint8_t *data = ...;
           int size = ...;
    
           while(size > 0))
           {
               AVPacket pkt;
               av_init_packet(&pkt);
               pkt.data  = data;
               pkt.size  = size;

               int frameFinished = 0;
               int bytesDecoded = avcodec_decode_video2(pCodecCtx, pSrcFrame,&frameFinished, &pkt);
               if(bytesDecoded > 0)
               {
                   data += bytesDecoded;
                   size -= bytesDecoded;
               }
               if(frameFinished)
               {
                   int numBytes =avpicture_get_size(pCodecCtx->pix_fmt,pCodecCtx->width,pCodecCtx->height);
                   uint8_t *pBuffer = new uint8_t[numBytes];
                   avpicture_fill((AVPicture *)pDstFrame, pBuffer,pCodecCtx->pix_fmt, pCodecCtx->width,pCodecCtx->height);
                   av_picture_copy((AVPicture *)pDstFrame, (AVPicture *)pSrcFrame,pCodecCtx->pix_fmt, pCodecCtx->width,pCodecCtx->height);
                   
                   // pBuffer/numBytes/pCodecCtx->pix_fmt :YUV/RGB数据
                   delete[]pBuffer;                   
               }
               
               if(bytesDecoded < 0)
                   break ;
           }
           av_free(pSrcFrame);
           av_free(pDstFrame);
       }
       else if(codecType == AVMEDIA_TYPE_AUDIO)
       {
           // 分配解码内存空间
           uint8_t *pBuffer = new uint8_t[AVCODEC_MAX_AUDIO_FRAME_SIZE];
    
           // 因为内存的原因，所以需要多分配一些数据, FF_INPUT_BUFFER_PADDING_SIZE
           uint8_t *data = ...;
           int size = ...;
    
           while(size > 0)
           {
               AVPacket pkt;
               av_init_packet(&pkt);
               pkt.data  = data;
               pkt.size  = size;
               
               int outSize = AVCODEC_MAX_AUDIO_FRAME_SIZE;
               int bytesDecoded = avcodec_decode_audio3(pCodecCtx, (int16_t*)pBuffer, &outSize, &pkt);
               if(bytesDecoded > 0)
               {
                   data += bytesDecoded;
                   size -= bytesDecoded;
               }
               if((bytesDecoded >= 0)&& (outSize >0))
               {
                   // pBuffer/outSize : PCM数据
                   // 格式
                   // pCodecCtx->channels;
                   // pCodecCtx->sample_fmt;
                   // pCodecCtx->sample_rate;
               }               
           }           
       }
       
       gMutexFFmpeg.lock();       
       // 关闭和释放
       avcodec_close(pCodecCtx);
       gMutexFFmpeg.unlock();
       av_free(pCodecCtx);
    }
    
   4)视音频编码(YUV/RGB=>Video ES, PCM=>AudioES)
    // 参考V3代码:media/videoencoder.cpp/audioencoder.cpp
    {
       // video encode
       avcodec_register_all();
       // 查找编码器
       AVCodec *avCodec =avcodec_find_encoder((CodecID)mConfig.codec);
       AVCodecContext *codecCtx = avcodec_alloc_context();
       codecCtx->codec_type   = AVMEDIA_TYPE_VIDEO;
       codecCtx->codec_id     = (CodecID)mConfig.codec;
       codecCtx->width        = mOutFormat.width;
       codecCtx->height       = mOutFormat.height;
       codecCtx->pix_fmt      = (PixelFormat)mOutFormat.pixelFormat;

       uint32 num = 0;
       uint32 den = 0;
       SampleUtil::FPS2Timescale(mOutFormat.frameRate, num, den);
       codecCtx->time_base.num = num;
       codecCtx->time_base.den =den;       
       codecCtx->bit_rate     = mConfig.bitRate*1000;
       codecCtx->max_b_frames  = 0;
       codecCtx->gop_size     = 100;
       if(codecCtx->codec_id == CODEC_ID_MPEG1VIDEO)
       {
           codecCtx->mb_decision = FF_MB_DECISION_RD;
       }
       else
       {
           codecCtx->mb_decision = FF_MB_DECISION_RD;
       }
       
       avcodec_open(codecCtx, avCodec);
       // 分配编码后的内存，分配为1MB
       mOutputBuffer.resize(1*1024*1024);
       
       AVFrame *pSrcFrame = avcodec_alloc_frame();
       
       avcodec_get_frame_defaults(pSrcFrame);
       int ret = avpicture_fill((AVPicture *)pSrcFrame, (uint8_t*)inData.data, (PixelFormat)mOutFormat.pixelFormat,mOutFormat.width, mOutFormat.height);

       AVRational time_base;
       time_base.num = 1;
       time_base.den = 1000000;
       pSrcFrame->pts = av_rescale_q(inData.dts, time_base,codecCtx->time_base);
       
       int bytesWritten = avcodec_encode_video(codecCtx, (uint8*)mOutputBuffer.data(), mOutputBuffer.size(),
           isEmpty ? NULL : pSrcFrame);

       outData.data  = (char*)mOutputBuffer.data();
       outData.size  = bytesWritten;
       outData.isKey =(mCodecCtx->coded_frame->key_frame !=0);
    
    
       av_free(pSrcFrame);
       avcodec_close(codecCtx);
       av_free(codecCtx);
       
       
       // audio encode请看audioencoder.cpp文件       
    }
    
   5)图像格式转换(YUV/RGB <=> YUV/RGB& Resize)
    // 参考代码:media/imageconverter.cpp
    {
       SwsContext *pSwsCtx = NULL;
       
       
       
       // resize 算法
       int swsFlags  = SWS_LANCZOS; //SWS_FAST_BILINEAR;
       // 初始化
       pSwsCtx = sws_getCachedContext(NULL, srcWidth, srcHeight,srcFmt,
           dstWidth, dstHeight, dstFmt, swsFlags, NULL, NULL, NULL);
       
       // 设置数据到结构 AVPicture
       AVPicture avSrcPic;
       AVPicture avDstPic;
       memset(&avSrcPic, 0, sizeof(avSrcPic));
       memset(&avDstPic, 0, sizeof(avDstPic));
       int dstRet = avpicture_fill(&avDstPic, (uint8_t*)pDstBuffer, dstFmt, dstWidth, dstHeight);
     
     {
       // pSrcBuffer - 源数据
       // pDstBuffer - 目标数据
       int srcRet = avpicture_fill(&avSrcPic, (uint8_t*)pSrcBuffer, srcFmt, srcWidth, srcHeight);

       // 执行转换
       sws_scale(pSwsCtx, avSrcPic.data, avSrcPic.linesize, 0,abs(srcHeight), avDstPic.data, avDstPic.linesize);
      }
      
       // 释放
       sws_freeContext(pSwsCtx);
    }
    
   6)封装格式(Muxer, .mp4/.avi/.mkv...)
    // 参考代码:interface IFileWriter, media/impl/filewriterimpl.cpp
    {
       av_register_all();

       AVFormatContext * pFormatCtx;
       avformat_alloc_output_context2(&pFormatCtx, NULL,"mp4", "c:\\out.mp4");
       
       {
           // new video stream
           AVStream * avStream = av_new_stream(pFormatCtx,pFormatCtx->nb_streams;
           avcodec_get_context_defaults3(avStream->codec,NULL);

           AVCodecContext *codecCtx = avStream->codec;
           codecCtx->codec_id      = (CodecID)format->codecId;
           codecCtx->codec_type    = AVMEDIA_TYPE_VIDEO;
           codecCtx->width         = format->width;
           codecCtx->height        = format->height;
           codecCtx->bit_rate      = 800000;
           uint32 num = 0;
           uint32 den = 0;
           SampleUtil::FPS2Timescale(format->frameRate, num,den);
           codecCtx->time_base.num  =num;
           codecCtx->time_base.den  =den;
           av_set_pts_info(streamInfo->avStream, 64, num,den);
           
           if(pFormatCtx->oformat->flags& AVFMT_GLOBALHEADER)
           {
               codecCtx->flags |= CODEC_FLAG_GLOBAL_HEADER;
           }
           switch(codecCtx->codec_id)
           {
           case CODEC_ID_H264:
               {
                   AVBitStreamFilterContext * avFilter =av_bitstream_filter_init("h264_mp4toannexb");
               }
               break ;
           case CODEC_ID_AAC:
               {
                   codecCtx->frame_size = 1024;
                   AVBitStreamFilterContext * avFilter =av_bitstream_filter_init("aac_adtstoasc");
               }
               break ;
           }
           // 设置解码相关数据, 比如H264要设置:SPS & PPS
           codecCtx->extradata_size = ;// size;
           codecCtx->extradata     = ;// (uint8_t *)av_malloc(size +FF_INPUT_BUFFER_PADDING_SIZE);
       }
       {
           // new stream
           AVStream * avStream = av_new_stream(pFormatCtx,pFormatCtx->nb_streams;
           avcodec_get_context_defaults3(avStream->codec,NULL);
       }
       
       err = av_set_parameters(pFormatCtx, NULL);
       // 以写的方式打开文件
       err = avio_open(&pFormatCtx->pb,"c:\\out.mp4", AVIO_FLAG_WRITE);

       // 写文件头信息
       err = av_write_header(pFormatCtx);
       
       {
           const AVRational in_time_base = { 1, 1000000 };
           AVRational out_time_base =avStream->time_base;

           AVPacket pkt = { 0 };
           av_init_packet(&pkt);
           
           pkt.stream_index = streamId; // 流的id
           pkt.data  = ;//(uint8_t*)mediaSample->data();
           pkt.size  =;//mediaSample->size();
           // 转换dts/pts时间单位 1/1000000=>avStream->time_base
           pkt.dts   =av_rescale_q(mediaSample->dts(), in_time_base,out_time_base);
           pkt.pts   =av_rescale_q(mediaSample->pts(), in_time_base,out_time_base);
           pkt.flags = mediaSample->isKey() ? AV_PKT_FLAG_KEY :0;

           // 写入一帧数据
           int err = av_interleaved_write_frame(pFormatCtx,&pkt);

           av_free_packet(&pkt);           
       }
               
       // 写文件尾信息
       av_write_trailer(pFormatCtx);
       
       // 释放
       // av_bitstream_filter_close(avFilter);
       avio_close(pFormatCtx->pb);
       avformat_free_context(pFormatCtx);
    }
    
   7)滤镜filter的使用(crop, resize, deinterlace, drawtext, overlay, vflip,...)
   通过搭建若干个filter可以对视音频进行一系列的处理.
       
    a).Simple filtergraphs:
    
        reencode filter graph:
        _________                       __________             ______________
       |        |                     |         |           |             |
       | decoded |  simple filtergraph | filtered | encoder   | encoded data |
       | frames  | -------------------> |frames   |---------> |packets     |
       |_________|                     |__________|           |______________|
       

       filter graph:
        _______       _____________       _______       _____       ________
       |      |     |            |     |      |     |    |     |       |
       | input | ---> | deinterlace | ---> |scale | ---> | fps | ---> | output|
       |_______|     |_____________|     |_______|     |_____|     |________|
       
       
            int ret =av_vsink_buffer_get_video_buffer_ref(mBufferDstCtx,&picRef, 0);
request_frame


start_frame
draw_slice
end_frame
      
      
    b).Complexfiltergraphs:
        _________
       |        |
       | input 0|\                   __________
       |_________|\                 |         |
                    \  _________    /|output 0 |
                     \|        |  / |__________|
        _________    \| complex | /
       |        |    |        |/
       | input 1 |---->| filter  |\
       |_________|    |        | \   __________
                      /| graph   |  \|         |
                     /|        |   \| output 1 |
        _________   / |_________|   |__________|
       |        | /
       | input 2 |/
       |_________|
       
    
    V3组件中实现的interface IVideoPreprocess代码示例:
   代码文件:media/impl/videopreprocessimpl.cpp
    
    我们搭建的filtergraph:
                                                                                        /1-->pad----\
       input-->deinterlace-->fps-->logoremove-->color-->imageoverlaps-->crop-->resize<-0----------->output
                                                                                        \2-->crop---/
                                                                                            
       {
           avcodec_register_all();
           avfilter_register_all();
           
           AVFilterGraph * pFilterGraph = NULL;
           AVFilterContext * pBufferSrcCtx = NULL;
           AVFilterContext * pBufferDstCtx = NULL;
           
           AVFrame * pSrcFrame   =avcodec_alloc_frame();
           AVFrame * pSinkFrame  =avcodec_alloc_frame();
           AVFrame * pDstFrame   =avcodec_alloc_frame();

           // 设定输出格式列表，我们仅支持PIX_FMT_YUV420P
           PixelFormat pix_fmts[] = { PIX_FMT_YUV420P, PIX_FMT_NONE };
           char args[512];
           
           AVFilterContext *lastFilterCtx = NULL;
           
           // 我们使用到的filter,其中"nl_"开头的是我们自己写的filter
           // 输入buffer filter
           AVFilter*bufferFilter    = avfilter_get_by_name("buffer");
           // deinterlace filter, 目前使用yadif filter
           AVFilter*yadifFilter     = avfilter_get_by_name("yadif");
           // 我们自己实现的fps转换filter
           AVFilter*fpsFilter       = avfilter_get_by_name("nl_fps");
           // 我们自己实现的遮logo的filter,支持多个，动态设置，能够设定区间范围
           AVFilter*delogosFilter   = avfilter_get_by_name("nl_delogos");
           // 我们自己实现的调节对比度和亮度的filter
           AVFilter*colorFilter     = avfilter_get_by_name("nl_color");
           // 我们自己实现的叠加图片的filter,支持多个，动态设置，能够设定区间范围
           AVFilter *overlaysFilter   =avfilter_get_by_name("nl_overlays");
           // crop filter
           AVFilter*cropFilter      = avfilter_get_by_name("crop");
           // resize filter
           AVFilter*resizeFilter    = avfilter_get_by_name("scale");
           // 图像扩展filter,可以在图像边界填充特定的颜色
           AVFilter*padFilter       = avfilter_get_by_name("pad");
           // 输出buffer filter
           AVFilter *buffersinkFilter =avfilter_get_by_name("buffersink");

           // 创建graph
           pFilterGraph = avfilter_graph_alloc();

           // 开始创建filter
           
           AVRational tb  = { 1, 1000000 };
           AVRational sar = { 0, 1 };
           // 计算图像宽度比
           av_reduce(&sar.num, &sar.den,mConfig.width, mConfig.height, 1000*1000);

           // 设定 buffer filter的参数
           //w:h:pixfmt:time_base.num:time_base.den:sample_aspect_ratio.num:sample_aspect_ratio.den:sws_param
           sprintf(args, "%d:%d:%d:%d:%d:%d:%d",
               mConfig.width, mConfig.height, mConfig.pixelFormat, tb.num, tb.den,sar.num, sar.den);

           // input filter
           err =avfilter_graph_create_filter(&pBufferSrcCtx, bufferFilter,  "in", args, NULL,pFilterGraph);
           // 记录前一个filter context
           lastFilterCtx = pBufferSrcCtx;
       
           // 如果需要 deinterlace,则创建 yadif filter,同时和前一个filter进行连接
           // deinterlace : yadif
           if(mConfig.deinterlace > 0)
           {
               if(yadifFilter == NULL)
                   break ;

               // yadif filter的参数
               // mode:parity
               sprintf(args, "%d:%d", 0, -1);

               // 创建filter,同时加入到graph
               AVFilterContext *deinterlaceCtx = NULL;
               err =avfilter_graph_create_filter(&deinterlaceCtx, yadifFilter, "yadif", args, NULL, pFilterGraph);
               if(err < 0)
                   break ;

               // 和前一个filter进行连接
               err = avfilter_link(lastFilterCtx, 0, deinterlaceCtx, 0);
               if(err < 0)
                   break ;

               lastFilterCtx = deinterlaceCtx;
           }
           // ... 中间略过
                   
           // 创建output filter
           err = avfilter_graph_create_filter(&pBufferDstCtx,buffersinkFilter, "out", NULL, pix_fmts, pFilterGraph);
           if(err < 0)
               break ;
    
           // 和前一个filter进行连接
           err = avfilter_link(lastFilterCtx, 0, pBufferDstCtx, 0);
           if(err < 0)
               break ;
               
           // 配置 graph
           err = avfilter_graph_config(pFilterGraph, NULL);

           
           // 把输入frame填充到结构AVFrame
           avpicture_fill((AVPicture *)pSrcFrame, (uint8_t*)inMediaSample->data(),
               (PixelFormat)mConfig.pixelFormat, mConfig.width,mConfig.height);
           pSrcFrame->width  =mConfig.width;
           pSrcFrame->height = mConfig.height;
           pSrcFrame->format = mConfig.pixelFormat;
           pSrcFrame->pts =inMediaSample->dts();

           // 开始写input写入frame
           ret = av_vsrc_buffer_add_frame(pBufferSrcCtx, pSrcFrame,AV_VSRC_BUF_FLAG_OVERWRITE);
           
           
           // 从输出filter查看输入是否可以获取数据,返回可获取的数目
           int count =avfilter_poll_frame(pBufferDstCtx->inputs[0]);
           if(count > 0)
           {
               AVFilterBufferRef *picRef = NULL;
               // 从输出filter中获取结果
               int ret = av_vsink_buffer_get_video_buffer_ref(pBufferDstCtx,&picRef, 0);
               if(picRef != NULL)
               {
                   // 转换AVFilterBufferRef到AVFrame
                   avfilter_fill_frame_from_video_buffer_ref(pSinkFrame,picRef);
                   pSinkFrame->format =picRef->format;
                   pSinkFrame->width  =picRef->video->w;
                   pSinkFrame->height =picRef->video->h;
                   
                   const int numBytes =avpicture_get_size((PixelFormat)pSinkFrame->format,pSinkFrame->width,pSinkFrame->height);
                   // 转换时间单位
                   AVRational tb  = { 1, 1000000 };
                   const int64 dts = av_rescale_q(picRef->pts,mBufferDstCtx->inputs[0]->time_base,tb);
                   // 获取图像数据
                   avpicture_fill((AVPicture *)pDstFrame, (uint8_t*)mediaSample->data(),
                       (PixelFormat)pSinkFrame->format,pSinkFrame->width,pSinkFrame->height);

                   av_picture_copy((AVPicture *)pDstFrame, (AVPicture*)pSinkFrame,
                       (PixelFormat)pSinkFrame->format,pSinkFrame->width,pSinkFrame->height);                       
                   
                   // 释放buffer计数器
                   avfilter_unref_buffer(picRef);
               }
           }
       }