FFmpeg+MFC动态桌面（二）

对于音视频文件的解码，ffmpeg无疑是很好的选择，对于ffmpeg雷神又不得不提

雷神博客地址：https://blog.csdn.net/leixiaohua1020/category_1843731.html

另附一个ffmpeg的简单demo地址：http://dranger.com/ffmpeg/tutorial02.html

最开始采用了最简单的音视频解码流程，将音频视频同时解码显示输出，但是当视频的分辨率达到1920*1080时，视频的解码转换速度开始变低，此时音频的卡顿听起来就很明显，然后不得已又将音频与视频的解码分开做缓冲处理，然后进行解码显示，卡顿问题解决，但是声话开始不同步，不得不再解决声画同步也就是音视频同步。

参考博客：https://www.cnblogs.com/wangguchangqing/p/5900426.html

本文中的流程图：

 下边上代码：

Media.h：

#include "Audio.h"
#include "Video.h"
class CMedia
{
public:
	CMedia();
	~CMedia();
private:
	HWND m_hwnd;
	CAudio *audio;
	CVideo *video;
	AVFormatContext	*pFormatCtx;//封装格式上下文结构体，保存视频文件封装格式相关内容
	SDL_Thread      *parse_tid; //解包线程id
	SDL_TimerID     timer_tid;  //定时器id
public:
	bool InitMedia(); //初始化FFMPEG以及SDL
	bool CloseInput();//关闭输入文件流

	bool OpenInput(const char* filepath); //打开输入文件流
	bool StartDecode(int w, int h);//开始解码视频流
	void SetVolume(std::function<int(void)> GetCurVolume);
	void SetPlayCB(std::function<void(AVFrame *pframe)> DisPlay);
private:
	bool MEDIA_QUIT;
	static int DecodeThread(void *data);
	static uint32_t SDLTimerCallback(uint32_t interval, void *opaque);//SDL定时器回调函数
	void ScheduleRefresh(int delay);//设置刷新帧定时器
	void VideoFrameRefresh();//计算时延刷新帧
};

media.cpp

#include "Media.h"

//音频队列容量上限
#define MAX_AUDIOQ_SIZE (5 * 16 * 1024)
//视频队列容量上限
#define MAX_VIDEOQ_SIZE (5 * 256 * 1024)


CMedia::CMedia() :MEDIA_QUIT(false), timer_tid(0),
pFormatCtx(nullptr), parse_tid(nullptr)
{
	//申请内存
	audio = new CAudio();
	video = new CVideo();
	InitMedia();
}


CMedia::~CMedia()
{
	CloseInput();
}

//初始化FFMPEG以及SDL
bool CMedia::InitMedia()
{
	av_register_all();			// 注册所有的codec 编解码器  
	//avformat_network_init();	//网络协议初始化类似于调用 WSAStartup()（windows特有）
	if (SDL_Init(SDL_INIT_VIDEO | SDL_INIT_AUDIO | SDL_INIT_TIMER))//初始化SDLSDL_INIT_EVERYTHING
	{
		//printf("Could not initialize SDL - %s\n", SDL_GetError());
		return false;
	}
	return true;
}

//打开输入文件流，获取媒体信息
bool CMedia::OpenInput(const char* filepath)
{
	if (avformat_open_input(&pFormatCtx, filepath, NULL, NULL) != 0) //打开输入流文件
	{
		//printf("Couldn't open input stream.\n");
		return false;
	}
	if (avformat_find_stream_info(pFormatCtx, NULL)<0)	//获取音视频流信息
	{
		//printf("Couldn't find stream information.\n");
		return false;
	}

	// 输出标准媒体流信息 
	//av_dump_format(pFormatCtx, 0, filepath, 0);

	if (!video->InitVideo(pFormatCtx))//初始化视频解码器
	{
		return false;
	}

	if (!audio->InitAudio(pFormatCtx))//初始化y解码器
	{
		return false;
	}

	return true;
}

//设置音量回调函数
void CMedia::SetVolume(std::function<int(void)> GetCurVolume)
{
	if (audio)audio->SetVolume(GetCurVolume);
	return;
}

void CMedia::SetPlayCB(std::function<void(AVFrame *pframe)> DisPlay)
{
	if (video)video->SetPlayCB(DisPlay);
	return;
}

//清理输入流
bool CMedia::CloseInput()
{	
	//移除定时器
	if (timer_tid)
	{	
		SDL_RemoveTimer(timer_tid);
		timer_tid = 0;
	}
	
	//停止音视频流解析
	MEDIA_QUIT = true;
	SDL_WaitThread(parse_tid, NULL);//停止解流	
	parse_tid = nullptr;

	//清理音视频解析线程
	if (audio)
	{	
		delete audio;
		audio = NULL; 
	}
	if (video)
	{ 
		delete video; 
		video = NULL; 
	}

	if (pFormatCtx)
	{
		avformat_close_input(&pFormatCtx);
		pFormatCtx = nullptr;
	}
	
	SDL_Quit();
	return true;
}

//开始解码视频流
bool CMedia::StartDecode(int w,int h)
{	
	// 创建解码线程，读取packet到队列中缓存
	parse_tid = SDL_CreateThread(DecodeThread, "parsepacket", this); 
	if (!parse_tid)
	{
		//printf("DecodeThread Creat failed.\n");
		return false;
	}

	
	//播放音频
	if (!audio->AudioPlay())
	{
		//printf("AudioPlay failed.\n");
		return false;
	}

	//播放视频
	if (!video->VideoPlay(w,h))
	{
		//printf("VideoPlay failed.\n");
		return false;
	}
	ScheduleRefresh(40);//定时刷新图像
	return true;
}


//解包线程
int CMedia::DecodeThread(void *data)
{
	CMedia* media = reinterpret_cast<CMedia*>(data);
	
	AVPacket *packet = (AVPacket *)av_malloc(sizeof(AVPacket));//初始化
	av_init_packet(packet);

	while (true)
	{
		if (media->MEDIA_QUIT)//退出解码线程
		{
			break;
		}
		//超出队列预定大小进行延迟等待
		if (media->audio->Que_audio.size > MAX_AUDIOQ_SIZE||media->video->Que_packet.size > MAX_VIDEOQ_SIZE)
		{
			SDL_Delay(10);
			continue;
		}

		int ret = av_read_frame(media->pFormatCtx, packet);//解码音视频流
		if (ret < 0)
		{
			if (ret == AVERROR_EOF)//文件末尾，结束线程
			{
				//播放完成
				while (media->audio->Que_audio.size > 0)//音频是按包解的，缓冲区小于0，则解析完成
				{
					SDL_Delay(10);
				}
				//通知完成播放
				SDL_Event event;
				event.type = FF_FINISH_EVENT;
				SDL_PushEvent(&event);
				break;
			}
				
			if (media->pFormatCtx->pb->error == 0) // No error,wait for user input
			{
				SDL_Delay(100);
				continue;
			}
			else
			{
				break;
			}
		}

		if (packet->stream_index == AVMEDIA_TYPE_AUDIO) // 音频流
		{
			media->audio->Que_audio.EnQueue(packet);
		}

		if (packet->stream_index == AVMEDIA_TYPE_VIDEO) // 视频流
		{
			media->video->Que_packet.EnQueue(packet);
		}
		av_packet_unref(packet);  
	}
	av_free_packet(packet);//释放包
	av_free(packet);

	return 0;
}

// 延迟delay毫秒后刷新video帧 (定时器)
void CMedia::ScheduleRefresh(int delay)
{
	timer_tid = SDL_AddTimer(delay, SDLTimerCallback, this);
	return;
}

//定时回调函数
uint32_t CMedia::SDLTimerCallback(uint32_t interval, void *opaque)
{
	CMedia *pthis = reinterpret_cast<CMedia*>(opaque);
	pthis->VideoFrameRefresh();
	return 0;  /* 0 means stop timer */
}


//计算时延刷新帧
void CMedia::VideoFrameRefresh()
{
	if (video->stream)//存在视频流
	{
		if (video->Que_frame.nb_frames == 0)
		{
			ScheduleRefresh(1);//随时准备更新图像
		}
		else
		{ 
			double actual_delay = video->CalcDelay([this]()->double{return audio->GetAudioClock();});	//计算时延
			if (actual_delay > 0)//取帧成功，得到下一帧延时
			{
				ScheduleRefresh(static_cast<int>(actual_delay * 1000 + 0.5));//设置下一帧的时延
				video->DisPlay();
			}	
		}
	}
	else
	{
		ScheduleRefresh(100);//延迟100ms
	}
	return;
}

video.h

#include "def.h"
#include "PacketQueue.h"
#include "FrameQueue.h"
class CVideo
{
public:
	CVideo();
	virtual ~CVideo();
	bool InitVideo(AVFormatContext	*pFormatCtx);//初始化视频流信息
	bool VideoPlay(int w,int h);
	void SetPlayCB(std::function<void(AVFrame *pframe)> RefreshPic);//回调显示图像
	double CalcDelay(std::function<double(void)> GetRefClock); //根据参考时钟计算时延
	void Clean();//清理视频解码设备
	void DisPlay();//对图像做缩变处理并显示
	
private:
	bool VIDEO_QUIT;//退出解包线程
	double Synchronize(AVFrame *srcFrame, double pts);//计算修正pts
	static int DecodePackt(void *data);//解包取帧线程
public:
	
	int stream_index;              //指示当前视频流信息
	AVCodecContext *video_ctx;     //获取编解码器相关信息
	AVStream *stream;			   //视频流信息	
	AVFrame *pFrame;
	AVFrame *displayFrame;
	
	CPacketQueue Que_packet;        //保存的video packet的队列缓存
	CFrameQueue Que_frame;          //保存解码后的原始帧数据
private:
	double frame_timer;          //Sync fields
	double frame_last_pts;
	double frame_last_delay;
	double video_clock;			//视频播放到当前帧时的已播放的时间长度
	SDL_Thread *decodepack_tid;	//解包线程
	std::function<void(AVFrame *pframe)> RefreshPic;
};

video.cpp

#include "Video.h"

static const double SYNC_THRESHOLD = 0.01;
static const double NOSYNC_THRESHOLD = 10.0;
static const double AV_SYNC_THRESHOLD_MAX = 0.05;//半秒钟即需要同步时钟

CVideo::CVideo() :stream_index(-1), video_ctx(nullptr), 
stream(nullptr), pFrame(nullptr), displayFrame(nullptr),
decodepack_tid(nullptr),frame_timer(0.0),frame_last_delay(0.0),
frame_last_pts(0.0), video_clock(0.0), VIDEO_QUIT(false)
{
	pFrame = av_frame_alloc();	//帧缓冲
	displayFrame = av_frame_alloc();
}

CVideo::~CVideo()
{
	Clean();
	return;
}
//初始化视频流信息
bool CVideo::InitVideo(AVFormatContext	*pFormatCtx)
{
	if (!pFormatCtx)
		return false;
	for (int i = 0; i < pFormatCtx->nb_streams; i++)
	{
		if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO)	//判断是否含视频
		{
			stream_index = i;
			break;
		}
	}
	if (stream_index<0)
	{
		//printf("can not find video stream \n");
		return false;
	}
	stream = pFormatCtx->streams[stream_index];
	//double video_duration = stream->duration * av_q2d(stream->time_base);
	//printf("video duration:%lf \n", video_duration);
	
	//视频解码器初始化	
	video_ctx = stream->codec; //获取编解码器相关信息
	AVCodec *pCodec_video = avcodec_find_decoder(video_ctx->codec_id);	 //查找编解码器

	if (pCodec_video == NULL)
	{
		//printf("Codec not found video_AVCodec.\n");
		return false;
	}
	if (avcodec_open2(video_ctx, pCodec_video, NULL)<0) //打开解码器并进一步完善编码器上下文相关信息以及各种分配内存检查参数
	{
		//printf("Could not open codec.\n");
		return false;
	}

	frame_timer = static_cast<double>(av_gettime()) / 1000000.0;
	frame_last_delay = 40e-3;
	return true;
}

//清理视频解码器
void CVideo::Clean()
{
	VIDEO_QUIT = true;

	Que_packet.Quit();//退出取包循环
	Que_frame.Quit();//退出取帧

	SDL_WaitThread(decodepack_tid,NULL);

	if (video_ctx)
	{	
		avcodec_flush_buffers(video_ctx);//冲洗缓冲
		avcodec_close(video_ctx);
	}
	if (pFrame)
		av_frame_free(&pFrame);
	if (displayFrame)
		av_frame_free(&displayFrame);	
	return;
}

//创建视频播放窗口
bool CVideo::VideoPlay(int w, int h)
{
	//初始化帧存储
	pFrame = av_frame_alloc();
	displayFrame = av_frame_alloc();//分配显示帧内存
	displayFrame->format = AV_PIX_FMT_RGB32;//AV_PIX_FMT_YUV420P;
	displayFrame->width = w;
	displayFrame->height = h;
	av_frame_get_buffer(displayFrame, 1);

	//包解帧线程
	decodepack_tid = SDL_CreateThread(DecodePackt, "de_v_p", this);
	if (!decodepack_tid)
	{
		//printf("video DecodePackt Thread Creat failed.\n");
		return false;
	}
	return true;
}

//修正pts的时间戳
double CVideo::Synchronize(AVFrame *srcFrame, double pts)
{
	if (pts != 0)
		video_clock = pts; // 获取到pts,将视频时延设置为pts
	else
		pts = video_clock; // 未获取到pts,将其近似为上一帧的pts

	double frame_delay = av_q2d(stream->codec->time_base);//需要求出扩展延时：
	frame_delay += srcFrame->repeat_pict * (frame_delay * 0.5); //repeat_pict / (2 * fps); 其中 fps = 1/frame_delay;
	video_clock += frame_delay;//这个延迟 是显示时间
	return pts;

}

//解帧线程
int CVideo::DecodePackt(void *data)
{
	CVideo *video = reinterpret_cast<CVideo*>(data);

	AVFrame *frame = av_frame_alloc();

	AVPacket packet;
	av_init_packet(&packet);
	double pts = 0.0;

	//将数据包还原成图像帧
	while (true)
	{
		//退出解包线程
		if (video->VIDEO_QUIT)
		{
		    break;
		}
		
		if (video->Que_frame.nb_frames >= CFrameQueue::capacity)//缓存大于30帧，延迟10ms解帧
		{
			SDL_Delay(10);
			continue;
		}

		video->Que_packet.DeQueue(&packet, true);

		int got_picture = 0;
		//解码一帧视频数据
		avcodec_decode_video2(video->video_ctx, frame, &got_picture, &packet);
	
		//计算pts 以流中的时间为基础预估的时间戳
		if ((pts = av_frame_get_best_effort_timestamp(frame)) == AV_NOPTS_VALUE)
			pts = 0.0;
		
		pts *= av_q2d(video->stream->time_base);//pts*时间基 = 该frame显示的时间戳。

		if (got_picture) //是否解码成功
		{

			//翻转图像
			frame->data[0] += frame->linesize[0] * (video->video_ctx->height - 1);
			frame->linesize[0] *= -1;
			frame->data[1] += frame->linesize[1] * (video->video_ctx->height / 2 - 1);
			frame->linesize[1] *= -1;
			frame->data[2] += frame->linesize[2] * (video->video_ctx->height / 2 - 1);
			frame->linesize[2] *= -1;


			pts = video->Synchronize(frame, pts);//修正pts
			frame->opaque = &pts;
			video->Que_frame.EnQueue(frame);//存入帧列
		}
		av_frame_unref(frame);
		av_packet_unref(&packet);
		av_free_packet(&packet);
	}
	av_frame_free(&frame);
	return 0;

}


//根据参考时钟计算时延
double CVideo::CalcDelay(std::function<double(void)> GetRefClock)
{
	if (!Que_frame.DeQueue(&pFrame))return -1;//获取一帧图像（要改）

	// 将视频同步到音频上，计算下一帧的延迟时间
	double current_pts = *(double*)pFrame->opaque;
	double delay = current_pts - frame_last_pts;
	if (delay <= 0 || delay >= 1.0)
		delay = frame_last_delay;

	frame_last_delay = delay;
	frame_last_pts = current_pts;

	// 当前显示帧的PTS来计算显示下一帧的延迟（diff < 0 => 减速 ,diff > 0 => 加速）
	double ref_clock = GetRefClock();
	double diff = current_pts - ref_clock;// 误差


#if Time//用于显示系统时间
	int h = (int)ref_clock / 3600;
	int m = ((int)ref_clock - (3600 * h)) / 60;
	int s = (int)ref_clock % 60;
	printf("%02d:%02d:%02d\n", h,m,s);
#endif

	double threshold = (delay > SYNC_THRESHOLD) ? delay : SYNC_THRESHOLD;//下限 0.01

	if (fabs(diff) < NOSYNC_THRESHOLD) // 不同步 调整
	{
		if (diff <= -threshold) // 慢了，delay设为0
			delay = 0;
		else if (diff >= threshold) // 快了，加倍delay
			delay *= 2;
	}
	frame_timer += delay;//总的时延

	double cur_timer = static_cast<double>(av_gettime()) / 1000000.0;//ms

	//计算当前真正的时延
	double actual_delay = frame_timer - cur_timer;

	if ((delay > 0) && ((cur_timer - frame_timer) > AV_SYNC_THRESHOLD_MAX))//计时 落后 系统时间超过半秒更新参考系统基准时间
	{
		frame_timer = cur_timer + delay;
		actual_delay = delay;
	}
	if (actual_delay <= 0.010)
		actual_delay = 0.010;

	return actual_delay;
}

//回调显示图像
void CVideo::SetPlayCB(std::function<void(AVFrame *pframe)> DisPlay)
{
	RefreshPic = DisPlay;
	return;
}

//对图像做缩变处理并显示
void CVideo::DisPlay()
{
	//缩变转换
	SwsContext *sws_ctx = sws_getContext(video_ctx->width, video_ctx->height, video_ctx->pix_fmt,
		displayFrame->width, displayFrame->height, AVPixelFormat(displayFrame->format), SWS_FAST_BILINEAR, NULL, NULL, NULL);

	sws_scale(sws_ctx, pFrame->data, pFrame->linesize, 0,video_ctx->height, displayFrame->data, displayFrame->linesize);

	sws_freeContext(sws_ctx);
	
	RefreshPic(displayFrame);//显示图片
	delete pFrame->opaque;
	av_frame_unref(pFrame);
	return ;
}

audio.h

#include "def.h"
#include "PacketQueue.h"

class CAudio
{
public:
	CAudio();
	virtual ~CAudio();
	bool InitAudio(AVFormatContext	*pFormatCtx);//初始化音频流信息
	bool AudioPlay();
	void SetVolume(std::function<int(void)> GetCurVolume);//设置音量回调函数
	double GetAudioClock();//获取音频时钟作为参考时钟
	void Clean();//清理解码器相关信息
private:
	static void AudioCallback(void* userdata, Uint8 *stream, int len);//回调函数（格式固定）
	int AudioDecodeFrame(uint8_t *audio_buf, int buf_size);//音频解包
	int AudioResampling(AVFrame * deframe,uint8_t *out_buf);//对音频进行转换重采样

private:
	bool AUDIO_QUIT;
	int stream_index;          //音频流标志
	AVCodecContext *audio_ctx; //编解码器相关信息
	AVStream *stream;		   //音频流信息
	AVFrame *pFrame;
	AVPacket pPacket;
public:
	CPacketQueue Que_audio;    //保存的video packet的队列缓存
private:
	std::function<int(void)> m_VolumeCb;  //回调函数
	const uint32_t BUFFER_SIZE; // 缓冲区的大小
	double audio_clock;			// 音频时间戳
	uint8_t *audio_buff;        // 解码后数据的缓冲空间
	uint32_t audio_buff_size;   // buf中的数据长度
	uint32_t audio_buff_index;  // 当前播放指针
};

audio.cpp

#include "Audio.h"
#define MAX_AUDIO_FRAME_SIZE 192000

CAudio::CAudio() :BUFFER_SIZE(MAX_AUDIO_FRAME_SIZE),
stream_index(-1), audio_buff(nullptr), AUDIO_QUIT(false),
pFrame(nullptr)
{
	audio_buff = (uint8_t*)av_malloc(MAX_AUDIO_FRAME_SIZE);
	memset(audio_buff, 0, MAX_AUDIO_FRAME_SIZE);
	pFrame = av_frame_alloc();
	av_init_packet(&pPacket);
}


CAudio::~CAudio()
{
	Clean();
	return;
}

void CAudio::Clean()
{	
	AUDIO_QUIT = true;
	Que_audio.Quit();//退出取包循环
	SDL_Delay(10);//等待10ms
	SDL_CloseAudio();
	if (audio_buff)
		av_free(audio_buff);
	if (audio_ctx)
		avcodec_close(audio_ctx);
	if (pFrame)
		av_frame_free(&pFrame);
	return;
}
bool CAudio::InitAudio(AVFormatContext	*pFormatCtx)//初始化音频流信息
{
	if (!pFormatCtx)
		return false;
	for (int i = 0; i < pFormatCtx->nb_streams; i++)
	{
		if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO)	//判断是否含视频
		{
			stream_index = i;
			break;
		}
	}
	if (stream_index<0)
	{
		//printf("can not find audio stream \n");
		return false;
	}
	stream = pFormatCtx->streams[stream_index];
	//double audio_duration = stream->duration * av_q2d(stream->time_base);
	//printf("audio duration:%lf \n",audio_duration);

	audio_ctx = pFormatCtx->streams[AVMEDIA_TYPE_AUDIO]->codec; //获取解码器上下文
	AVCodec *pCodec_audio = avcodec_find_decoder(pFormatCtx->streams[stream_index]->codec->codec_id);
	if (pCodec_audio == NULL)
	{
		//printf("Codec not found audio_AVCode.\n");
		return false;
	}
	if (avcodec_open2(audio_ctx, pCodec_audio, NULL)<0) 
	{
		//printf("Could not open audio codec.\n");
		return false;
	}

	return true;
}

//设置音量回调函数
void CAudio::SetVolume(std::function<int(void)> GetCurVolume)
{
	m_VolumeCb = GetCurVolume;
	return;
}

//获取当前音频播放的pts
/*
用audio缓冲区中剩余的数据除以每秒播放的音频数据得到剩余数据的播放时间，
从Audio clock中减去这部分的值就是当前的audio的播放时长。
*/
double CAudio::GetAudioClock()
{
	int hw_buf_size = audio_buff_size - audio_buff_index;//剩余数据长度
	int bytes_per_sec = stream->codec->sample_rate * audio_ctx->channels * 2;//每秒播放数据长度
	double pts = audio_clock - static_cast<double>(hw_buf_size) / bytes_per_sec;//当前pts 单位 s
	return pts;
}

bool CAudio::AudioPlay()
{
	SDL_AudioSpec desired;
	desired.freq = audio_ctx->sample_rate;
	desired.channels = audio_ctx->channels;
	desired.format = AUDIO_S16SYS;
	desired.samples = 1024; //samples: AAC-1024 MP3-1152
	desired.silence = 0;
	desired.userdata = this;
	desired.callback = AudioCallback;

	if (SDL_OpenAudio(&desired, nullptr) < 0)
	{
		return false;
	}
	SDL_PauseAudio(0); // playing
	return true;
}

//向设备发送audio数据的回调函数（格式固定）
void CAudio::AudioCallback(void* userdata, Uint8 *stream, int len)
{
	CAudio *audio = (CAudio*)userdata;

	SDL_memset(stream, 0, len);

	int audio_size = 0;
	int len1 = 0;
	while (len > 0)// 向设备发送长度为len的数据
	{
#if 1
		//退出
		if (audio->AUDIO_QUIT)return;

		if (audio->audio_buff_index >= audio->audio_buff_size) // 缓冲区中无数据
		{
			// 从packet中解码数据
			audio_size = audio->AudioDecodeFrame(audio->audio_buff, sizeof(audio->audio_buff));
			if (audio_size < 0) // 没有解码到数据或出错，填充0
			{
				audio->audio_buff_size = 0;
				memset(audio->audio_buff, 0, audio->audio_buff_size);
			}
			else
			{
				audio->audio_buff_size = audio_size;
			}

			audio->audio_buff_index = 0;//读指针
		}
		len1 = audio->audio_buff_size - audio->audio_buff_index; // 缓冲区中剩下的数据长度
		if (len1 > len) // 向设备发送的数据长度为len
			len1 = len;

		SDL_MixAudio(stream, audio->audio_buff + audio->audio_buff_index, len, audio->m_VolumeCb());

		len -= len1;
		stream += len1;
		audio->audio_buff_index += len1;
#endif
	}

}

//对音频包进行解帧
int CAudio::AudioDecodeFrame(uint8_t *audio_buf, int buf_size)
{
	int data_size = 0;
	static int audio_pkt_size = 0;	
	//在调用avcodec_decode_audio4函数后的返回值就是从流中解了的数据的数量，
	//如果返回值和传进去的AVPacket * pkt的size值不一样，说明没解完，这个包里有多个AVFrame。
	for (;;)
	{
		if (AUDIO_QUIT)//退出
		{
			av_frame_unref(pFrame);
			return -1;
		}
			
		while (audio_pkt_size>0)
		{
			int got_picture = 0;
			int readsize = avcodec_decode_audio4(audio_ctx, pFrame, &got_picture, &pPacket);
			if (readsize < 0)//空包
			{
				audio_pkt_size = 0;
				break;
			}
			//剩余包长
			audio_pkt_size -= readsize;

			if (got_picture)//获取到一帧数据
			{
				data_size = AudioResampling(pFrame, audio_buf);
			}

			if (data_size <= 0)//解析到空帧，继续解析
			{
				continue;
			}
			av_frame_unref(pFrame);//释放帧内存
			return data_size;//先返回一帧数据，然后回来继续解帧
		}

		if (pPacket.data)
		{
		  av_packet_unref(&pPacket);//释放包内存
		}
		if (!Que_audio.DeQueue(&pPacket, true))
		{
			return -1;
		}	

		if (pPacket.pts != AV_NOPTS_VALUE)
		{
			audio_clock = av_q2d(stream->time_base) * pPacket.pts;
		}
		audio_pkt_size = pPacket.size;	
	}
	return 0;

}

//音频重采样
int CAudio::AudioResampling(AVFrame * deframe, uint8_t *out_buf)
{
	AVSampleFormat dst_format = AV_SAMPLE_FMT_S16;
	int64_t channel_layout = av_get_default_channel_layout(audio_ctx->channels);
	SwrContext * swr_ctx = swr_alloc();
	swr_ctx = swr_alloc_set_opts(swr_ctx, channel_layout, dst_format,
		audio_ctx->sample_rate, channel_layout, audio_ctx->sample_fmt,
		audio_ctx->sample_rate, NULL, NULL);


	if (!swr_ctx || swr_init(swr_ctx) < 0)//参数设置失败
	{
		//printf("Failed to initialize the resampling context.\n");
		return -1;
	}

	// 计算转换后的sample个数 a * b / c
	uint64_t dst_nb_samples = av_rescale_rnd(swr_get_delay(swr_ctx, deframe->sample_rate) + deframe->nb_samples, audio_ctx->sample_rate, deframe->sample_rate, AV_ROUND_UP);
	
	// 转换，返回值为转换后的sample个数 
	int nb = swr_convert(swr_ctx, &out_buf, static_cast<int>(dst_nb_samples), (const uint8_t**)deframe->data, deframe->nb_samples);
	
	//样本所占字节大小
	int data_size = audio_ctx->channels * nb * av_get_bytes_per_sample(dst_format);
	
	//由于一个packet中可以包含多个帧，packet中的PTS比真正的播放的PTS可能会早很多，
	//可以根据Sample Rate 和 Sample Format来计算出该packet中的数据可以播放的时长，
	//再次更新Audio clock 。（audio_clock内的时长是先于当前pts的，所以计算时要减去剩余数据的播放时长）
	// 每秒钟音频播放的字节数 sample_rate * channels * sample_format(一个sample占用的字节数)
	audio_clock += static_cast<double>(data_size) / (2 * stream->codec->channels * stream->codec->sample_rate);
	swr_free(&swr_ctx);//释放转换设备
	
	return data_size;
}

PacketQueue.h

#ifndef _PACKETQUEUE_H_
#define _PACKETQUEUE_H_
#include <queue>
#include"def.h"
class CPacketQueue
{
public:
	CPacketQueue();
	~CPacketQueue();

	bool EnQueue(const AVPacket *packet);//进队
	bool DeQueue(AVPacket *packet, bool block);//出队

	void Quit();//发送信号退出出队循环
	Uint32   nb_packets;//队列元素个数
	Uint32   size; //队列数据大小

private:
	bool quit;
	std::queue<AVPacket> queue;
	SDL_mutex *mutex;
	SDL_cond  *cond;
};

#endif

PacketQueue.cpp

#include "PacketQueue.h"

CPacketQueue::CPacketQueue() :nb_packets(0), size(0), quit(false)
{
	mutex = SDL_CreateMutex();
	cond = SDL_CreateCond();
}


CPacketQueue::~CPacketQueue()
{
	while (!queue.empty())
	{
		AVPacket pkt = queue.front();
		//引用计数减1（数据域的引用计数减为0时会自动释放）
		av_packet_unref(&pkt);
		queue.pop();
	}
	SDL_DestroyMutex(mutex);//释放锁
	SDL_DestroyCond(cond);//释放条件变量
}
bool CPacketQueue::EnQueue(const AVPacket *packet)
{
	AVPacket pkt;
	av_init_packet(&pkt);//初始化
	if (av_packet_ref(&pkt, packet) < 0)//拷贝packet
	{
		//printf("av_packet_ref failed\n");
		return false;
	}

	SDL_LockMutex(mutex);//加锁
	queue.push(pkt);

	size += pkt.size;
	nb_packets++;

	SDL_CondSignal(cond);
	SDL_UnlockMutex(mutex);
	return true;

}
bool CPacketQueue::DeQueue(AVPacket *packet, bool block)
{
	bool ret = false;
	SDL_LockMutex(mutex);
	while (true)
	{
		if (quit)
		{
			ret = false;
			break;
		}

		if (!queue.empty())
		{
			if (av_packet_ref(packet, &queue.front()) < 0)
			{
				ret = false;
				break;
			}
			AVPacket pkt = queue.front();
			queue.pop();

			av_packet_unref(&pkt);

			nb_packets--;
			size -= packet->size;
			ret = true;
			break;
		}
		else if (!block)
		{
			ret = false;
			break;
		}
		else
		{
			SDL_CondWait(cond, mutex);
		}
	}
	SDL_UnlockMutex(mutex);
	return ret;
}

//发送信号退出出队循环
void CPacketQueue::Quit()
{
	quit = true;
	SDL_CondSignal(cond);
	return;
}

FrameQueue.h

#ifndef _CFRAMEQUEUE_H_
#define _CFRAMEQUEUE_H_
#include <queue>
#include "def.h"
class CFrameQueue
{
public:
	CFrameQueue();
	~CFrameQueue();
	bool EnQueue(const AVFrame* frame);
	bool DeQueue(AVFrame **frame);
	void Quit();
	static const int capacity = 30;//缓存30帧图像
	uint32_t nb_frames;//帧数
private:
	std::queue<AVFrame*> queue;//缓冲队列
	SDL_mutex* mutex;
	SDL_cond * cond;
	bool quit;
	
};
#endif

FrameQueue.cpp

#include "FrameQueue.h"

CFrameQueue::CFrameQueue() :nb_frames(0), quit(false)
{
	mutex = SDL_CreateMutex();
	cond = SDL_CreateCond();
}


CFrameQueue::~CFrameQueue()
{
	while(!queue.empty())//释放队列剩余数据
	{
		auto tmp = queue.front();
		delete tmp->opaque;
		av_frame_free(&tmp);
		queue.pop();	
	}
	SDL_DestroyMutex(mutex);//释放锁
	SDL_DestroyCond(cond);//释放条件变量
}


bool CFrameQueue::EnQueue(const AVFrame* frame)
{
	AVFrame* p = av_frame_alloc();

	int ret = av_frame_ref(p, frame);
	if (ret < 0)
	{
		//printf("av_frame_ref failed\n");
		return false;
	}

	//上一个指向的是一个局部的变量，这里重新分配pts空间	
	p->opaque = (void *)new double(*(double*)p->opaque); 

	SDL_LockMutex(mutex);//入队 //这里是否考虑率加入缓冲帧数限制 暂时先这样
	queue.push(p);
	nb_frames++;
	SDL_CondSignal(cond);
	SDL_UnlockMutex(mutex);
	return true;
}
bool CFrameQueue::DeQueue(AVFrame **frame)
{
	bool ret = true;

	SDL_LockMutex(mutex);
	while (true)
	{
		if (quit)
		{
			ret = false;
			break;
		}
		if (!queue.empty())
		{
			if (av_frame_ref(*frame, queue.front()) < 0)
			{
				ret = false;
				break;
			}

			auto tmp = queue.front();
			queue.pop();

			av_frame_free(&tmp);

			nb_frames--;

			ret = true;
			break;
		}
		else
		{
			SDL_CondWait(cond, mutex);
		}
	}

	SDL_UnlockMutex(mutex);
	return ret;
}

//发送信号退出出队循环
void CFrameQueue::Quit()
{
	quit = true;
	SDL_CondSignal(cond);
	return;
}

以上音频视频解码同步的核心部分了。