使用ffmpeg + d3d/gdi 播放视频

前段时间，公司的一个项目需要一个rtsp的播放库，原本打算直接用vlc播放的，但我觉得vlc太庞大了，很多功能没必要，还不如用ffmpeg+d3d简单的实现一个库，因此就有了今天讲的这个东西。一个解码库，分为三个部分：网络，解码，显示。网络和解码在ffmpeg里带了，直接用就好，显示，用d3d直接显示yuv是最佳方案了。整个库采用多线程模型，播放一路就创建一个播放线程。库的接口如下：

struct hvplayer;
typedef struct hvplayer hvplayer;
/**
 *播放退出回调
 *@param h 播放器指针
 *@param state 退出状态
 */
typedef void (*playend_cb)(hvplayer *h);


enum player_state
{
    PLAYER_CONNECTING=0,
    PLAYER_PLAYING,
    PLAYER_OFF
};



/**
 *初始化一个播放器
 *@param hwnd 一个表示画面显示区域的id,windows上为窗口句柄
 *@param url rtsp地址
 *@return NULL失败,>0为一个播放器指针
 */
HVEXP hvplayer *hvplayer_new(int32_t hwnd, const char *url);


/**
 *播放一个rtsp视频,该方法是一个非阻塞函数,内部会创建一个线程去执行播放
 *任务，自动无限重连，直到调用hvplayer_close
 *
 *@param h 一个指向播放器的指针
 *@return 成功为0,失败-1
 */
 HVEXP int hvplayer_play(hvplayer *h);

/**
 *获取播放器的状态,成功会设置state
 *
 *@param h 一个指向播放器的指针
 *@return player_state枚举的值
 */
 HVEXP int hvplayer_getstate(hvplayer *h);



/**
 *停止播放,并结束播放线程,该方法会阻塞至播放线程结束,同时释放hvplayer句柄
 *
 *@param h 一个指向播放器的指针
 */
 HVEXP void hvplayer_close(hvplayer *h);


/**
 *注册播放结束回调,hvplayer_close后回调会被调用
 *
 *@param h 一个指向播放器的指针
 *@param cb 具体见回调定义
 *@waring 多次注册会覆盖
 *@waring 不要阻塞该调用线程
 */
 HVEXP void hvplayer_set_endcb(hvplayer *h, playend_cb cb);
 /************************************************************************/
 /*SDK初始化,必须在使用SDK之前初始化
  *@return 0成功
 /************************************************************************/
 HVEXP int hvdevicevideo_init(void);

使用起来很简单，一个hvplayer对象对应一路视频，先hvplayer_new(),在hvplayer_play(),最后hvplayer_close()就好了。库使用前要调用hvdevicevideo_init进行初始化.
hvplayer的定义如下：

 1 struct hvplayer
 2 {
 3     playend_cb end;
 4     int32_t hwnd;
 5     int32_t flage;
 6     char *url;
 7     enum player_state ste;
 8     HANDLE thread;
 9     clock_t pretm;
10     int play;
11 };

flage是控制重连循环的，play是控制帧数据读取循环的,pretm是控制rtsp服务器连接超时的.

play_loop是播放线程的函数，外层是重连循环,_do是实际播放循环.

 1 static int play_loop(void* p)
 2 {
 3     hvplayer *h=(hvplayer*)p;
 4     while(h->flage)
 5     {
 6         h->ste=PLAYER_CONNECTING;
 7         h->pretm=clock();
 8         _do(h);
 9         int s=TIMEOUT_S-(clock()-h->pretm)/CLOCKS_PER_SEC;
10         if(s>0) Sleep(s*1000);
11     }
12     if(h->end) h->end(h);
13     free(h->url);
14     free(h);
15     return 1;
16 }

 

 1 static void _do(hvplayer*h)
 2 {
 3     AVCodec *codec=NULL;
 4     AVCodecContext *cc=NULL;
 5     AVPacket pk={0};
 6     AVFormatContext *afc=NULL;
 7     int vindex=-1;
 8     afc=avformat_alloc_context();
 9     if(afc==0){
10         goto err;
11     }
12     afc->interrupt_callback.callback=timeoutcheck;
13     afc->interrupt_callback.opaque=h;
14     AVDictionary *dir=NULL;
15     char *k1="stimeout";
16     char *v1="10";
17     char *k2="rtsp_transport";
18     char *v2="tcp";
19     char *k4="max_delay";
20     char *v4="50000";
21     int r=av_dict_set(&dir,k1,v1,0);
22     r=av_dict_set(&dir,k2,v2,0);
23     av_dict_set(&dir,k4,v4,0);
24     if(avformat_open_input(&afc,h->url,NULL,&dir)) {
25         goto err;
26     }
27     if(avformat_find_stream_info(afc,NULL)<0) {
28         goto err;
29     }
30     for(int i=0;i<(int)afc->nb_streams;i++)
31     {
32         if(afc->streams[i]->codec->codec_type==AVMEDIA_TYPE_VIDEO)
33         {
34             vindex=i;
35             break;
36         }
37     }
38     if(vindex==-1){
39         goto err;
40     }
41     cc=afc->streams[vindex]->codec;
42     codec=avcodec_find_decoder(cc->codec_id);
43     
44     if(codec==0) goto end;
45     if(avcodec_open2(cc,codec,NULL)<0) {
46         goto err;
47     }
48     int dr;
49     hvframe frame;
50     render *render=render_create(RENDER_TYPE_D3D,h->hwnd,cc->coded_width,cc->coded_height);
51     if(render==NULL) 
52     {
53         render=render_create(RENDER_TYPE_GDI,h->hwnd,cc->coded_width,cc->coded_height);
54         if(render==NULL){
55             goto end;
56         }
57     }
58     AVFrame *yuv_buf=av_frame_alloc();
59     if(yuv_buf==0) {
60         goto end;
61     }
62     h->ste=PLAYER_PLAYING;
63     DWORD a,b;
64     while (h->flage&&av_read_frame(afc,&pk)>=0)
65     {
66         
67         if(pk.stream_index==vindex)
68         {
69                 avcodec_decode_video2(cc,yuv_buf,&dr,&pk);
70                 if(dr>0)
71                 {
72                     frame.h=cc->coded_height;
73                     frame.y=yuv_buf->data[0];
74                     frame.u=yuv_buf->data[1];
75                     frame.v=yuv_buf->data[2];
76                     frame.ypitch=yuv_buf->linesize[0];
77                     frame.uvpitch=yuv_buf->linesize[1];
78                     render->draw(render,&frame);
79                 }
80         }
81         av_free_packet(&pk);
82     }
83 end:
84     if(yuv_buf) av_frame_free(&yuv_buf);
85     if(render) render->destory(&render);
86 err:
87     if(cc) avcodec_close(cc);
88     if(afc){
89         avformat_close_input(&afc);
90         avformat_free_context(afc);
91     }
92 }

 

1. avformat_alloc_context（）分配AVFormatContext对象。
2. avformat_open_input（）打开一个流媒体源，可以是文件，rtsp,这是一个阻塞函数，知道解析成功或失败才返回.那如何设置超时时间呢？这个对象提供了两个中断回调，解析时ffmpeg会以一定频率调用这个回调，这个回调的返回值影响avformat_open_input（）是否立马返回.这个回调函数指针就是AVFormatContext->interrupt_callback.callback,同时可以用->interrupt_callback.opaque绑定一个用户数据.回调函数返回1时，avformat_open_input（）会失败返回,返回0则正常运行.

          我是这样处理的(为了6秒超时判定):

 1 #define  TIMEOUT_S 6
 2 
 3 int timeoutcheck(void *p)
 4 {
 5     hvplayer *h=(hvplayer*)p;
 6     if(h->flage==0) return 1;
 7     if(h->ste==PLAYER_CONNECTING)
 8     {
 9         clock_t ctm=clock();
10         int s=(ctm-h->pretm)/CLOCKS_PER_SEC;
11         if(s>=TIMEOUT_S)
12             return 1;        
13     }
14     return 0;
15 }

　　3、avformat_find_stream_info()解析流的格式信息，再用以下代码获取AVCodecContext:

 1 for(int i=0;i<(int)afc->nb_streams;i++)
 2     {
 3         if(afc->streams[i]->codec->codec_type==AVMEDIA_TYPE_VIDEO)
 4         {
 5             vindex=i;
 6             break;
 7         }
 8     }
 9     if(vindex==-1){
10         goto err;
11     }
12     cc=afc->streams[vindex]->codec;

　　4、avcodec_find_decoder(),利用AVCodecContext->codec_id创建AVCodec,再用avcodec_open2()打开解码器.

　　5、循环调用av_read_frame()获取一帧编码数据，再调用avcodec_decode_video2(AVCodec*,AVFrame*,int*,AVPacket*)解码为yuv,第三个参数标示是否解码成功(>0成功),最后调用显示模块显示即可.

 

显示上，我将其封装为render对象，分别实现了d3d,gdi.render的结构如下：

1 struct render{
2     int hwnd;
3     void (*draw)(struct render *self,hvframe *frame);
4     void (*destory)(struct render **self);
5 };
6 typedef struct render render;

d3d实现如下：

 1 #include <stdlib.h>
 2 #include "hvtype.h"
 3 #include "irender.h"
 4 #include "winapi.h"
 5 struct render_d3d
 6 {
 7     render base;
 8     IDirect3D9 *d3d;
 9     IDirect3DDevice9 *d3d_dev;
10     IDirect3DSurface9 *surface;
11     RECT rec;
12 };
13 
14 void d3d_draw(render *h,hvframe *frame);
15 void render_free(render **h);
16 
17 render *d3d_new(int hwnd,int pic_w,int pic_h)
18 {
19     struct render_d3d *result=(struct render_d3d *)malloc(sizeof(*result));
20     result->base.hwnd=hwnd;
21     result->d3d=Direct3DCreate9(D3D_SDK_VERSION);
22     if(result->d3d==NULL) return NULL;
23     D3DPRESENT_PARAMETERS d3dpp; 
24     memset(&d3dpp,0,sizeof(d3dpp));
25     d3dpp.Windowed=TRUE;
26     d3dpp.SwapEffect=D3DSWAPEFFECT_DISCARD;
27     d3dpp.BackBufferFormat=D3DFMT_UNKNOWN;
28     GetClientRect((HWND)result->base.hwnd,&result->rec);
29     HRESULT re=IDirect3D9_CreateDevice(result->d3d,D3DADAPTER_DEFAULT,D3DDEVTYPE_HAL,(HWND)result->base.hwnd,
30         D3DCREATE_SOFTWARE_VERTEXPROCESSING,&d3dpp,&result->d3d_dev);
31     if(FAILED(re)) return NULL;
32     re=IDirect3DDevice9_CreateOffscreenPlainSurface(result->d3d_dev,pic_w,pic_h,
33         (D3DFORMAT)MAKEFOURCC('Y','V','1','2'),
34         D3DPOOL_DEFAULT,&result->surface,NULL);
35     if(FAILED(re)) return NULL;
36     result->base.draw=d3d_draw;
37     result->base.destory=render_free;
38     return (render *)result;
39 }
40 
41 void d3d_draw(render *h,hvframe *frame)
42 {
43     struct render_d3d *self=(struct render_d3d*)h;
44     D3DLOCKED_RECT texture;
45     HRESULT re=IDirect3DSurface9_LockRect(self->surface,&texture,NULL,D3DLOCK_DONOTWAIT);
46     if(FAILED(re)) return;
47     int uvstep=texture.Pitch/2;
48     char *dest=(char*)texture.pBits;
49     char *vdest=(char*)texture.pBits+frame->h*texture.Pitch;
50     char *udest=(char*)vdest+frame->h/2*uvstep;
51     int uvn=0;
52     for(int i=0;i<frame->h;i++)
53     {
54         memcpy(dest,frame->y,frame->ypitch);
55         frame->y+=frame->ypitch;
56         dest+=texture.Pitch;
57     }
58     for(int i=0;i<frame->h/2;i++)
59     {
60         memcpy(vdest,frame->v,frame->uvpitch);
61         memcpy(udest,frame->u,frame->uvpitch);
62         vdest+=uvstep;
63         udest+=uvstep;
64         frame->v+=frame->uvpitch;
65         frame->u+=frame->uvpitch;
66     }
67     IDirect3DSurface9_UnlockRect(self->surface);
68     IDirect3DDevice9_BeginScene(self->d3d_dev);
69     IDirect3DSurface9 *back_surface;
70     IDirect3DDevice9_GetBackBuffer(self->d3d_dev,0,0,D3DBACKBUFFER_TYPE_MONO,&back_surface);
71     IDirect3DDevice9_StretchRect(self->d3d_dev,self->surface,NULL,back_surface,&self->rec,D3DTEXF_LINEAR);
72     IDirect3DDevice9_EndScene(self->d3d_dev);
73     IDirect3DDevice9_Present(self->d3d_dev,NULL,NULL,NULL,NULL);
74 }
75 void render_free(render **h)
76 {
77     if((*h)==0)return;
78     struct render_d3d *self=(struct render_d3d*)(*h);
79     IDirect3DDevice9_Clear(self->d3d_dev,0, NULL, D3DCLEAR_TARGET, D3DCOLOR_XRGB(0,0,0), 1.0f, 0 );
80     IDirect3DDevice9_Present(self->d3d_dev,NULL,NULL,NULL,NULL);
81     IDirect3DSurface9_Release(self->surface);
82     IDirect3DDevice9_Release(self->d3d_dev);
83     IDirect3D9_Release(self->d3d);
84     free(*h);
85     *h=0;
86 }

都是标准的固定管线步骤，就不赘述了，唯一要注意的就是yuv数据填充到surface中时要注意行对齐.

gdi实现如下：

 1 #include <stdlib.h>
 2 #include <windows.h>
 3 #include "hvpfconvert.h"
 4 #include "irender.h"
 5 #include "winapi.h"
 6 
 7 struct render_gdi
 8 {
 9     render base;
10     BITMAPINFO  bmp;
11     HDC dc;
12     hvpfconvert *convert;
13     uint8_t *buf;
14     RECT rec;
15 };
16 
17 
18 void gdi_free(render **h)
19 {
20     if((*h)==0)return;
21     struct render_gdi *self=(struct render_gdi*)(*h);
22     FillRect(self->dc,&self->rec,(HBRUSH)(pGetStockObject(BLACK_BRUSH)));
23     ReleaseDC((HWND)self->base.hwnd,self->dc);
24     hvpfconvert_free(&self->convert);
25     free(*h);
26     *h=0;
27 }
28 
29 void gdi_draw(render *h,hvframe *frame)
30 {
31     struct render_gdi *self=(struct render_gdi*)h;
32     hvpfconvert_convert2(self->convert, frame, self->buf);
33     pSetStretchBltMode(self->dc, STRETCH_HALFTONE);
34     pStretchDIBits(self->dc, 0, 0, self->rec.right, self->rec.bottom, 0, 0,
35         self->bmp.bmiHeader.biWidth, frame->h, self->buf, &self->bmp, DIB_RGB_COLORS, SRCCOPY);
36 }
37 
38 render *gdi_new(int32_t hwnd,int pic_w,int pic_h)
39 {
40     struct render_gdi *re = (struct render_gdi *)calloc(1,sizeof(*re));
41     re->convert = hvpfconvert_new(pic_w, pic_h, pic_w, pic_h, PF_YUV420P, PF_BGR24);
42     re->dc = GetDC((HWND)hwnd);
43     re->base.hwnd = hwnd;
44     GetClientRect((HWND)re->base.hwnd, &re->rec);
45     re->buf = (uint8_t*)malloc(hvpfconvert_get_size(pic_w, pic_h, PF_RGB24));
46     re->bmp.bmiHeader.biBitCount = 24;
47     re->bmp.bmiHeader.biClrImportant = BI_RGB;
48     re->bmp.bmiHeader.biClrUsed = 0;
49     re->bmp.bmiHeader.biCompression = 0;
50     re->bmp.bmiHeader.biHeight = -pic_h;
51     re->bmp.bmiHeader.biWidth = pic_w;
52     re->bmp.bmiHeader.biPlanes = 1;
53     re->bmp.bmiHeader.biSize = sizeof(re->bmp.bmiHeader);
54     re->bmp.bmiHeader.biSizeImage = pic_w*pic_h * 3;
55     re->bmp.bmiHeader.biXPelsPerMeter = 0;
56     re->bmp.bmiHeader.biYPelsPerMeter = 0;
57     re->base.destory=gdi_free;
58     re->base.draw=gdi_draw;
59     return (render *)re;
60 }

gdi是利用StretchDIBits函数显示位图，因此要将yuv转换为rgb数据，图像格式转换，ffmpeg也自带了高效的转换函数:sws_scale().

最后是sdk的初始化，hvdevicevideo_init()我调用了ffmpeg库和d3d所需的初始化函数：

1 int hvdevicevideo_init(void){
2     if(hv_winapi_init()) return -1;
3     avcodec_register_all();
4     av_register_all();
5     avformat_network_init();
6     sdk_init=1;
7     CoInitializeEx(NULL,COINIT_MULTITHREADED);
8     return 0;
9 }

最后整个库，性能还不错，不必vlc差。在支持d3d的i5机器上解码一路1080p视频占3%-5%左右的cpu.不支持的将启用gdi显示，每路占13%的cpu.可优化点是rtsp，ffmpeg的rtsp效率
一般般，基本有400毫秒左右的延迟，而liv555可以做到200的延迟，不过我的需求400延迟可以接受，因此就没去折腾了。后来公司有一嵌入式的项目，我又做了一版linux的实现，用了英特尔的vaapi驱动进行硬解码(因为项目运行的机器是一款atom的cpu,主频只有1.2g,软解一路要占40%多的cpu),后续再写基于ffmpeg的vaapi硬解码播放吧.