音视频开发系列（15）：视频与音频同步

上次分享了将视频与音频同时推流到服务上的代码封装，然后上节分享在测试后会发现音视频不同步，这边说一下原因：

从帧率及采样率，即可知道视频/音频播放速度。声卡和显卡均是以一帧数据来作为播放单位，如果单纯依赖帧率及采样率来进行播放，在理想条件下，应该是同步的，不会出现偏差。
以一个44.1KHz的AAC音频流和24FPS的视频流为例：
一个AAC音频frame每个声道包含1024个采样点，则一个frame的播放时长(duration)为：(1024/44100)×1000ms = 23.22ms；一个视频frame播放时长(duration)为：1000ms/24 = 41.67ms。理想情况下，音视频完全同步。

但实际情况下，如果用上面那种简单的方式，慢慢的就会出现音视频不同步的情况，要不是视频播放快了，要么是音频播放快了。可能的原因如下：

1、一帧的播放时间，难以精准控制。音视频解码及渲染的耗时不同，可能造成每一帧输出有一点细微差距，长久累计，不同步便越来越明显。（例如受限于性能，42ms才能输出一帧）

2、音频输出是线性的，而视频输出可能是非线性，从而导致有偏差。

3、媒体流本身音视频有差距。（特别是TS实时流，音视频能播放的第一个帧起点不同）。

由于会存在以上原因，所以最终会导致音视频推送不同步。

要做到音视频同步，主要有以下三种方法：

1、将视频同步到音频上：就是以音频的播放速度为基准来同步视频。
2、将音频同步到视频上：就是以视频的播放速度为基准来同步音频。
3、将视频和音频同步外部的时钟上：选择一个外部时钟为基准，视频和音频的播放速度都以该时钟为标准。

本次分享的主要是以第三种策略来进行同步，即采用外部时钟的方法。

主要思路大概如下：

1.首先开启视频和音频的线程，上次介绍过线程的策略为：音频利用qt的读取音频的接口，然后在在开启的线程中一直往一个链表中写入数据，视频利用的opencv的接口，然后在线程中的策略与音频的一致，都是利用一个链表来读写音视频数据，这边需要注意的地方是需要利用锁的机制，控制不能同时读写链表。

2.然后就是初始化音视频编解码器，配置音视频编解码器所需要的参数，配置的参数可以具体看我分享的代码。

3.初始化输出封装器，添加视频和音频流，这边主要用于音视频推流。

4.记录当前的时间并将之前在线程中存储的音视频数据进行清除，以便同步更为准确。

5.进入死循环读取音视频帧进行推流。这边会记录下当前的时间，传入每一帧的数据，规定每一帧的pts。这边的时间基数time_base都设置为1/1000000，即设置单位为微秒。

大概流程如上所示。

这边会出现一个问题，由于音频数据会出现两次的pts时间相同，这时候会报错，这边进行的处理是如果出现相同的时间的情况，就在上一次记录的时间的基础上加上1.2ms。

下面是代码的分享，里面有注释，如果需要工程的话可以在评论区留言，看到的话会进行发送的。

main.cpp

#include <QtCore/QCoreApplication>
#include <QThread>
#include <iostream>
#include "XMediaEncode.h"
#include "XRtmp.h"
#include "XAudioRecord.h"
#include "XVideoCapture.h"
using namespace std;
int main(int argc, char *argv[])
{
	QCoreApplication a(argc, argv);

	int ret = 0;
	char *outUrl = "rtmp://192.168.198.128/live";
	int sampleRate = 44100;
	int channels = 2;
	int sampleByte = 2;
	int nbSample = 1024;
	
	///打开摄像机
	XVideoCapture *xv = XVideoCapture::Get();
	if (!xv->Init(0))
	{
		cout << "open camera failed!" << endl;
		getchar();
		return -1;
	}
	cout << "open camera success!" << endl;
	xv->Start();

	///1 qt音频开始录制 
	XAudioRecord *ar = XAudioRecord::Get();
	ar->sampleRate = sampleRate;
	ar->channels = channels;
	ar->sampleByte = sampleByte;
	ar->nbSample = nbSample;
	if (!ar->Init())
	{
		cout << "XAudioRecord Init failed!" << endl;
		getchar();
		return -1;
	}
	ar->Start();
	///初始化格式上下文
	XMediaEncode *xe = XMediaEncode::Get();
	xe->inWidth = xv->width;
	xe->inHeight = xv->height;
	xe->outHeight = xv->height;
	xe->outWidth = xv->width;
	if (!xe->InitScale())
	{
		cout << "InitScale Init failed!" << endl;
		getchar();
		return -1;
	}
	cout << "初始化视频像素转换上下文成功！" << endl;

	///音频重采样
	xe->channels = channels;
	xe->nbSample = 1024;
	xe->sampleRate = sampleRate;
	xe->inSampleFmt = XSampleFMT::X_S16;
	xe->outSampleFmt = XSampleFMT::X_FLTP;
	if (!xe->InitResample())
	{
		getchar();
		return -1;
	}


	///4 初始化音频编码器
	if (!xe->InitAudioCodec())
	{
		getchar();
		return -1;
	}

	///4 初始化视频编码器
	if (!xe->InitVideoCodec())
	{
		getchar();
		return -1;
	}
	///5 封装器和音频流配置
	//a.创建输出封装器上下文

	XRtmp *xr = XRtmp::Get(0);
	if (!xr->Init(outUrl))
	{
		getchar();
		return -1;
	}
	int vindex = 0;
	vindex = xr->AddStream(xe->vc);
	//b.添加视频流
	if (vindex<0)
	{
		getchar();
		return -1;
	}
	int aindex = 0;
	aindex = xr->AddStream(xe->ac);
	//b.添加视频流
	if (aindex<0)
	{
		getchar();
		return -1;
	}
	//从编码器复制参数

	///6 打开rtmp的网络输出io
	//写入封装头

	if (!xr->SendHead())
	{
		getchar();
		return -1;
	}
	//一次读取一帧音频的字节数
	int readSize = xe->nbSample*channels*sampleByte;
	char *buf = new char[readSize];

	ar->Clear();
	xv->Clear();
	long long beginTime = GetCurTime();


	for (;;)
	{

		//一次读取一帧音频
		XData ad = ar->Pop();
		XData vd = xv->Pop();
		if (ad.size <= 0&&vd.size<=0)
		{
			QThread::msleep(1);
			continue;
		}
		if (ad.size > 0)
		{
			ad.pts = ad.pts - beginTime;
			//已经读取一帧源数据
			//重采样数据
			XData pcm = xe->Resample(ad);
			ad.Drop();
			XData pkt = xe->EncodeAudio(pcm);
			if (pkt.size>0)
			{
				//推流
				if (xr->SendFrame(pkt,aindex))
				{
					cout << "#" << flush;
				}
			}
		}
		if (vd.size > 0)
		{
			vd.pts = vd.pts - beginTime;
			XData yuv = xe->RGBToYUV(vd);
			vd.Drop();
			XData pkt = xe->EncodeVideo(yuv);
			if (pkt.size>0)
			{
				//推流		
				if (xr->SendFrame(pkt,vindex))
				{
					cout << "@" << flush;
				}
			}
		}

	}
	delete buf;
	getchar();
	return a.exec();
}

XAudioRecord.h

#pragma once
#include "XDataThread.h"
enum XAUDIOTYPE
{
	X_AUDIO_QT
};

class XAudioRecord:public XDataThread
{
public:
	int sampleRate = 44100;//样本率
	int channels = 2;//声道数
	int sampleByte = 2;//样本大小
	int nbSample = 1024;//一帧音频每个通道的样本数量
	static XAudioRecord *Get(XAUDIOTYPE type= X_AUDIO_QT,unsigned char index = 0);

	//开始录制
	virtual bool Init()=0;

	//停止录制
	virtual void Stop()=0;

	virtual ~XAudioRecord();
protected:
	XAudioRecord();
	
};

XAudioRecord.cpp

#include "XAudioRecord.h"
#include <QAudioInput>
#include <iostream>
#include <list>

using namespace std;
class CXAudioRecord :public XAudioRecord
{
public:
	void run()
	{
		cout << "进入音频录制线程" << endl;
		int readSize = nbSample*channels*sampleByte;
		char *buf = new char[readSize];
		while (!isExit)
		{
			//读取已录制音频
			//一次读取一帧音频
			if (input->bytesReady() < readSize)
			{
				QThread::msleep(1);
				continue;
			}

			int size = 0;
			while (size != readSize)
			{
				int len = io->read(buf + size, readSize - size);
				if (len < 0)break;
				size += len;
			}
			if (size != readSize)
			{
				continue;
			}
			long long pts = GetCurTime();
			//已读取一帧音频
			Push(XData(buf,readSize,pts));
		}
		delete buf;
		cout << "退出音频录制线程" << endl;

	}
	//开始录制
	bool Init()
	{
		Stop();
		///1 qt音频开始录制 
		QAudioFormat fmt;
		int ret;
		//采样频率
		fmt.setSampleRate(sampleRate);
		//通道数量
		fmt.setChannelCount(channels);
		//样本大小
		fmt.setSampleSize(sampleByte * 8);
		//格式
		fmt.setCodec("audio/pcm");
		//字节序
		fmt.setByteOrder(QAudioFormat::LittleEndian);
		fmt.setSampleType(QAudioFormat::UnSignedInt);
		QAudioDeviceInfo info = QAudioDeviceInfo::defaultInputDevice();
		if (!info.isFormatSupported(fmt))
		{
			cout << "Audio format not support!" << endl;
			fmt = info.nearestFormat(fmt);
		}
		cout << "Audio format success" << endl;

		input = new QAudioInput(fmt);
		//开始录制音频
		io = input->start();
		if (!io)return false;
		return true;
	}

	//停止录制
	virtual void Stop()
	{
		XDataThread::Stop();
		if (input)
			input->stop();
		if (io)
			io->close();
		input = NULL;
		io = NULL;

	}
	QAudioInput *input = NULL;
	QIODevice *io = NULL;
};

XAudioRecord *XAudioRecord::Get(XAUDIOTYPE type, unsigned char index)
{
	static CXAudioRecord record[255];
	return &record[index];
}
XAudioRecord::XAudioRecord()
{
}


XAudioRecord::~XAudioRecord()
{
}

XData.h

#pragma once
class XData
{
public:
	char *data = 0;
	int size = 0;
	long long pts = 0;
	void Drop();
	XData();
	//创建空间，并复制data内容
	XData(char *data,int size,long long p=0);
	~XData();
};

//获取当前时间戳(us)
long long GetCurTime();

XData.cpp

#include "XData.h"
#include <stdlib.h>
#include <string.h>
extern "C"
{
#include <libavutil/time.h>
}
XData::XData(char *data, int size, long long p)
{
	this->data = new char[size];
	memcpy(this->data, data, size);
	this->size = size;
	this->pts = p;
}
void XData::Drop()
{
	if (data)
		delete data;
	data = 0;
	size = 0;
}
XData::XData()
{
}


XData::~XData()
{
}

long long GetCurTime()
{
	return av_gettime();
}

XDataThread.h

#pragma once

#include <QThread>
#include "XData.h"
#include <list>
class XDataThread : public QThread
{
public:
	//（缓冲列表大小）列表最大值，超出删除最旧的数据
	int maxList = 100;
	//在列表结尾插入
	virtual void Push(XData d);
	//读取列表中最早的数据,返回数据需要调用Xdata.Drop清理
	virtual XData Pop();
	//启动线程
	virtual bool Start();
	//退出线程，并等待线程退出(阻塞)
	virtual void Stop();

	virtual void Clear();
	XDataThread();
	virtual ~XDataThread();
protected:
	//存放交互数据 插入策略 先进先出
	std::list<XData> datas;
	//互斥访问 datas
	QMutex mutex;
	//交互数据列表大小
	int dataCount = 0;
	//处理线程退出
	bool isExit = false;
};

XDataThread.cpp

#include "XDataThread.h"
void XDataThread::Clear()
{
	mutex.lock();
	while (!datas.empty())
	{
		datas.front().Drop();
		datas.pop_front();
	}
	mutex.unlock();
}

//在列表结尾插入
void XDataThread::Push(XData d)
{
	mutex.lock();
	//超出最大大小
	if (datas.size()>maxList)
	{
		datas.front().Drop();
		datas.pop_front();
	}
	datas.push_back(d);
	mutex.unlock();
}
//读取列表中最早的数据
XData XDataThread::Pop()
{
	mutex.lock();
	if (datas.empty())
	{
		mutex.unlock();
		return XData();
	}
	XData d = datas.front();
	datas.pop_front();
	mutex.unlock();
	return d;
}
//启动线程
bool XDataThread::Start()
{
	isExit = false;
	QThread::start();
	return true;
}
//退出线程，并等待线程退出(阻塞)
void XDataThread::Stop()
{
	isExit = true;
	wait();
}
XDataThread::XDataThread()
{
}


XDataThread::~XDataThread()
{
}

XMediaEncode.h

#pragma once
class AVCodecContext;
enum XSampleFMT
{
	X_S16 = 1,
	X_FLTP = 8
};
#include <XData.h>
///音视频编码接口类
class XMediaEncode
{
public:

	///输入参数
	int inWidth = 1280;
	int inHeight = 720;
	int inPixSize = 3;
	int channels = 2;
	int sampleRate = 44100;
	XSampleFMT inSampleFmt = X_S16;
	///输出参数
	int outWidth = 1280;
	int outHeight = 720;
	int bitrate = 4000000; 
	int fps = 25;
	int nbSample = 1024;
	XSampleFMT outSampleFmt = X_FLTP;

	//工厂的生产方法
	static XMediaEncode *Get(unsigned char index = 0);
	//初始化像素格式转换的上下文
	virtual	bool InitScale() = 0;

	//音频重采样上下文初始化
	virtual bool InitResample() = 0;


	virtual XData Resample(XData d) = 0;

	virtual XData RGBToYUV(XData rgb) = 0;

	//视频编码器初始化
	virtual bool InitVideoCodec() = 0;

	//音频编码器初始化
	virtual bool InitAudioCodec() = 0;

	//视频编码 返回值无需调用者清理
	virtual XData EncodeVideo(XData frame) = 0;
	
	//音频编码 返回值无需调用者清理
	virtual XData EncodeAudio(XData frame) = 0;

	virtual ~XMediaEncode();
	AVCodecContext *vc = 0; 		//编码器上下文
	AVCodecContext *ac = 0;         //音频编码器上下文
protected:
	XMediaEncode();
};

XMediaEncode.cpp

#include "XMediaEncode.h"
#include <iostream>
extern "C"
{
#include <libswscale/swscale.h>
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include "libswresample/swresample.h"
}
#pragma comment(lib,"swscale.lib")
#pragma comment(lib,"avcodec.lib")
#pragma comment(lib,"avutil.lib")
#pragma comment(lib,"swresample.lib")
using namespace std;

#if defined WIN32 || defined _WIN32
#include <windows.h>
#endif
//获取CPU数量
static int XGetCpuNum()
{
#if defined WIN32 || defined _WIN32
	SYSTEM_INFO sysinfo;
	GetSystemInfo(&sysinfo);

	return (int)sysinfo.dwNumberOfProcessors;
#elif defined __linux__
	return (int)sysconf(_SC_NPROCESSORS_ONLN);
#elif defined __APPLE__
	int numCPU = 0;
	int mib[4];
	size_t len = sizeof(numCPU);

	// set the mib for hw.ncpu
	mib[0] = CTL_HW;
	mib[1] = HW_AVAILCPU;  // alternatively, try HW_NCPU;

						   // get the number of CPUs from the system
	sysctl(mib, 2, &numCPU, &len, NULL, 0);

	if (numCPU < 1)
	{
		mib[1] = HW_NCPU;
		sysctl(mib, 2, &numCPU, &len, NULL, 0);

		if (numCPU < 1)
			numCPU = 1;
	}
	return (int)numCPU;
#else
	return 1;
#endif
}

class CXMediaEncode :public XMediaEncode
{
public:
	void Close()
	{
		if (vsc)
		{
			sws_freeContext(vsc);
			vsc = NULL;
		}
		if (yuv)
		{
			av_frame_free(&yuv);
		}
		if (asc)
		{
			swr_free(&asc);
		}
		if (vc)
		{
			avcodec_free_context(&vc);
		}
				if (vc)
		{
			avcodec_free_context(&vc);
		}
		if (pcm)
		{
			av_frame_free(&pcm);
		}
		vpts = 0;
		apts = 0;
		av_packet_unref(&vpack);
		av_packet_unref(&apack);
	}
	bool InitVideoCodec()
	{
				///4.初始化编码上下文
		//a 找到编码器
		if (!(vc = CreateCodec(AV_CODEC_ID_H264)))
		{
			return false;
		}

		//vc->codec_id = codec->id;
		vc->bit_rate = 50 * 1024 * 8; //压缩后每秒视频的bit位大小  200kB
		vc->width = outWidth;
		vc->height = outHeight;
		//vc->time_base = { 1,fps };
		vc->framerate = { fps,1 };

		//画面组的大小，多少帧一个关键帧
		vc->gop_size = 50;
		//设置不需要b帧
		vc->max_b_frames = 0;
		//设置像素格式
		vc->pix_fmt = AV_PIX_FMT_YUV420P;
		//d 打开编码器上下文
		int ret = avcodec_open2(vc, 0, 0);
		if (ret != 0)
		{
			char buf[1024] = { 0 };
			av_strerror(ret, buf, sizeof(buf) - 1);
			cout << buf << endl;
			return false;
		}
		cout << "avcodec_open2 success!" << endl;
		return true;
	}


	bool InitAudioCodec()
	{
		///4 初始化音频编码器
		if (!(ac=CreateCodec(AV_CODEC_ID_AAC)))
		{
			return false;
		}

		//音频的参数
		ac->bit_rate = 40000;
		ac->sample_rate = sampleRate;
		ac->sample_fmt = AV_SAMPLE_FMT_FLTP;
		ac->channels = channels;
		ac->channel_layout = av_get_default_channel_layout(channels);
		
		//打开编码器
		return OpenCodec(&ac);
	}
	XData EncodeVideo(XData frame)
	{
		av_packet_unref(&vpack);

		XData r;
		if (frame.size <= 0 || !frame.data)return r;

		AVFrame *p = (AVFrame *)frame.data;

//		frame->pts = vpts;
//		vpts++;
		int ret = avcodec_send_frame(vc, p);
		if (ret != 0)
		{
			return r;
		}

		ret = avcodec_receive_packet(vc, &vpack);
		if (ret != 0 || vpack.size < 0)
		{
			return r;
		}
		r.data = (char *)&vpack;
		r.size = vpack.size;
		r.pts = frame.pts;
		return r;
	}
	long long lasta = -1;

	XData EncodeAudio(XData frame)
	{
		XData r;
		if (frame.size <= 0 || !frame.data)return r;

		AVFrame *p = (AVFrame *)frame.data;
		if (lasta == p->pts)
		{
			p->pts += 1200;
		}
		lasta = p->pts;
		int ret = avcodec_send_frame(ac, p);
		if (ret != 0)return r;

		av_packet_unref(&apack);
		ret = avcodec_receive_packet(ac, &apack);
		//cout << "avcodec_receive_packet   " << ret << endl;
		if (ret != 0)return r;
		r.data = (char *)&apack;
		r.size = apack.size;
		r.pts = frame.pts;
		return r;
	}
	bool InitScale()
	{
		///2.初始化格式转换上下文
		vsc = sws_getCachedContext(vsc,
			//源宽、高、像素格式
			inWidth, inHeight, AV_PIX_FMT_BGR24,
			//目标宽、高、像素格式
			outWidth, outHeight, AV_PIX_FMT_YUV420P,
			SWS_BICUBIC,  //尺寸变化使用的算法
			0, 0, 0
			);

		if (!vsc)
		{
			cout << "sws_getCachedContext failed";
			return false;
		}

		///3.输出的数据结构
		yuv = av_frame_alloc();
		yuv->format = AV_PIX_FMT_YUV420P;
		yuv->width = inWidth;
		yuv->height = inHeight;
		yuv->pts = 0;
		//分配yuv空间
		int ret = av_frame_get_buffer(yuv, 32);
		if (ret != 0)
		{
			char buf[1024] = { 0 };
			av_strerror(ret, buf, sizeof(buf) - 1);
			throw exception(buf);
		}
		return true;
	}

	bool InitResample()
	{
		asc = NULL;
		asc = swr_alloc_set_opts(asc,
			av_get_default_channel_layout(channels), (AVSampleFormat)outSampleFmt, sampleRate,               //输出格式
			av_get_default_channel_layout(channels), (AVSampleFormat)inSampleFmt, sampleRate,//输入格式	
			0, 0);
		if (!asc)
		{
			cout << "swr_alloc_set_opts failed!" << endl;
			return false;
		}
		int ret = swr_init(asc);
		if (ret != 0)
		{
			char err[1024] = { 0 };
			av_strerror(ret, err, sizeof(err) - 1);
			cout << err << endl;
			return false;
		}
		cout << "音频重采样 上下文初始化成功" << endl;
		
		///3 音频重采样输出空间分配
		pcm = av_frame_alloc();
		pcm->format = outSampleFmt;
		pcm->channels = channels;
		pcm->channel_layout = av_get_default_channel_layout(channels);
		pcm->nb_samples = nbSample;//一帧音频一通道的采样数量
		ret = av_frame_get_buffer(pcm, 0); //给pcm分配存储空间
		if (ret != 0)
		{
			char err[1024] = { 0 };
			av_strerror(ret, err, sizeof(err) - 1);
			cout << err << endl;
			return false;
		}
		return true;
	}
	XData RGBToYUV(XData d)
	{
		XData r;
		r.pts = d.pts;
		///rgb to yuv
		//3.初始化输入的数据结构
		uint8_t *indata[AV_NUM_DATA_POINTERS] = { 0 };
		//indata[0] bgrbgrbgr
		//plane  indata[0] bbbbb indata[1]ggggg indata[2] rrrrr
		indata[0] = (uint8_t*)d.data;
		int insize[AV_NUM_DATA_POINTERS] = { 0 };
		//一行(宽)数据的字节数
		insize[0] = inWidth*inPixSize;
		int h = sws_scale(vsc, indata, insize, 0, inHeight,//源数据
			yuv->data, yuv->linesize);
		if (h <= 0)
		{
			return r;
		}
		yuv->pts = d.pts;
		r.data = (char *)yuv;
		int *p = yuv->linesize;
		while ((*p))
		{
			r.size += (*p)*outHeight;
			p++;
		}
		return r;
	}
	XData Resample(XData d)
	{
		XData r;
		const uint8_t *indata[AV_NUM_DATA_POINTERS] = { 0 };
		indata[0] = (uint8_t *)d.data;
		//已经读取一帧源数据
		//重采样数据
		int len = swr_convert(asc, pcm->data, pcm->nb_samples,//输出参数,输出存储地址，样本数
			indata, pcm->nb_samples
			);
		if (len <= 0)
		{
			return r;
		}
		//从采集就记录pts
		pcm->pts = d.pts;
		r.data = (char *)pcm;
		r.size = pcm->nb_samples*pcm->channels * 2;
		r.pts = d.pts;
		return r;
	}
private:
	bool OpenCodec(AVCodecContext **c)
	{
		int ret = avcodec_open2(*c, 0, 0);
		if (ret != 0)
		{
			char err[1024] = { 0 };
			av_strerror(ret, err, sizeof(err) - 1);
			cout << err << endl;
			avcodec_free_context(c);
			return false;
		}
		cout << "avcodec_open2 success!" << endl;
		return true;
	}
	AVCodecContext * CreateCodec(AVCodecID cid)
	{
		///4 初始化编码器
		AVCodec *codec = avcodec_find_encoder(cid);
		if (!codec)
		{
			cout << "avcodec_find_encoder failed!" << endl;
			return NULL;
		}
		//音频编码器上下文
		AVCodecContext * c = avcodec_alloc_context3(codec);
		if (!c)
		{
			cout << "avcodec_alloc_context3 failed!" << endl;
			return NULL;
		}
		cout << "avcodec_alloc_context3 success!" << endl;

		c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
		c->thread_count = XGetCpuNum();
		c->time_base = { 1,1000000 };
		return c;
	}
	SwsContext *vsc = NULL;         //像素格式转换上下文
	AVFrame *yuv = NULL;            //输出的yuv
	SwrContext *asc = NULL;         //音频重采样上下文
	AVPacket vpack = {0};           //视频帧
	AVPacket apack = { 0 };			//音频帧
	AVFrame *pcm = NULL;            //重采样输出的pcm
	int vpts = 0;
	int apts = 0;
}; 

XMediaEncode *XMediaEncode::Get(unsigned char index)
{
	static bool isFirst = true;
	if (isFirst)
	{
		//注册所有的编解码器
		avcodec_register_all();
		isFirst = false;
	}
	static CXMediaEncode cxm[255];
	return &cxm[index];
}
XMediaEncode::XMediaEncode()
{
}


XMediaEncode::~XMediaEncode()
{
}

XRtmp.h

#pragma once
class AVCodecContext;
class AVPacket;
#include "XData.h"
class XRtmp
{
public:
	static XRtmp *Get(unsigned char index = 0);
	//初始化封装器上下文
	virtual bool Init(const char *url) = 0;

	//添加视频或者音频流 失败返回-1 成功返回流索引
	virtual int AddStream(const AVCodecContext *c) = 0;
	
	//打开rtmp网络IO，发送封装头
	virtual bool SendHead() = 0;

	//rtmp 帧推流
	virtual bool SendFrame(XData d,int streamIndex=0) = 0;
	~XRtmp();
protected:
	XRtmp();
};

XRtmp.cpp

#include "XRtmp.h"
#include <iostream>
#include <string>
using namespace std;
extern "C"
{
#include <libavformat/avformat.h>
}
#pragma comment(lib,"avformat.lib")
class CXRtmp :public XRtmp
{
public:

	void Close()
	{
		if (ic)
		{
			avformat_close_input(&ic);
			vs = NULL;
		}
		vc = NULL;
		url = "";
	}
	bool Init(const char *url)
	{
		//a.创建输出封装器上下文
		int ret = avformat_alloc_output_context2(&ic, 0, "flv", url);
		this->url = url;
		if (ret != 0)
		{
			char buf[1024] = { 0 };
			av_strerror(ret, buf, sizeof(buf) - 1);
			cout << buf << endl;
			return false;
		}
		return true;
	}
	int AddStream(const AVCodecContext *c)
	{
		if (!c)return -1;
		AVStream *st = avformat_new_stream(ic, NULL);
		if (!st)
		{
			cout << ("avformat_new_stream failed!") << endl;
			return -1;
		}
		st->codecpar->codec_tag = 0;
		//从编码器复制参数
		avcodec_parameters_from_context(st->codecpar, c);
		av_dump_format(ic, 0, url.c_str(), 1);
		if (c->codec_type == AVMEDIA_TYPE_VIDEO)
		{
			vc = c;
			vs = st;
		}
		else if (c->codec_type == AVMEDIA_TYPE_AUDIO)
		{
			ac = c;
			as = st;
		}
		return st->index;
	}
	bool SendHead()
	{
		int ret = avio_open(&ic->pb, url.c_str(), AVIO_FLAG_WRITE);
		if (ret != 0)
		{
			char buf[1024] = { 0 };
			av_strerror(ret, buf, sizeof(buf) - 1);
			cout << buf << endl;
			return false;
		}
		//写入封装头
		ret = avformat_write_header(ic, NULL);
		if (ret != 0)
		{
			char buf[1024] = { 0 };
			av_strerror(ret, buf, sizeof(buf) - 1);
			cout << buf << endl;
			return false;
		}
		return true;
	}

	bool SendFrame(XData d, int streamIndex)
	{
		if (!d.data || d.size <= 0 )return false;
		AVPacket *pack = (AVPacket *)d.data;
		//判断是音频还是视频
		pack->stream_index = streamIndex;
		AVRational stime;
		AVRational dtime;

		if (as && ac &&pack->stream_index == as->index)
		{
			stime = ac->time_base;
			dtime = as->time_base;
		}
		else if (vs && vc && pack->stream_index == vs->index)
		{
			stime = vc->time_base;
			dtime = vs->time_base;

		}
		else
		{
			return false;
		}
		//推流
		pack->pts = av_rescale_q(pack->pts, stime, dtime);
		pack->dts = av_rescale_q(pack->dts, stime, dtime);
		pack->duration = av_rescale_q(pack->duration, stime, dtime);
		int ret = av_interleaved_write_frame(ic, pack);
		if (ret == 0)
		{
			cout << "#" << flush;
			return true;
		}
		return false;
	}
private:
	AVFormatContext *ic = NULL;  	//rtmp flv 封装器
	string url = "";
	//视频编码器流
	const AVCodecContext *vc = NULL;
	//视频流
	AVStream *vs = NULL;
	//音频编码器流
	const AVCodecContext *ac = NULL;
	//音频流
	AVStream *as = NULL;
};

XRtmp * XRtmp::Get(unsigned char index)
{
	static CXRtmp cxr[255];
	static bool isFirst = true;
	if (isFirst)
	{
		//注册所有的封装器
		av_register_all();
		//注册所有的网络协议
		avformat_network_init();
		isFirst = false;
	}
	return &cxr[index];
}
XRtmp::XRtmp()
{
}


XRtmp::~XRtmp()
{
}

XVideoCapture.h

#pragma once
#include "XDataThread.h"
class XVideoCapture:public XDataThread
{
public:
	int width = 0;
	int height = 0;
	int fps = 0;
	static XVideoCapture *Get(unsigned char index = 0);

	virtual bool Init(int camIdex = 0) = 0;
	virtual bool Init(const char *url) = 0;
	virtual void Stop() = 0;

	virtual ~XVideoCapture();
protected:
	XVideoCapture();

};

XVideoCapture.cpp

#include "XVideoCapture.h"
#include <opencv2/highgui.hpp>
#include <iostream>
using namespace cv;
using namespace std;
#pragma comment(lib,"opencv_world320.lib")

class CXVideoCapture:public XVideoCapture
{
public:
	 VideoCapture cam;
	 void run()
	 {
		 Mat frame;
		 cout << "进入视频线程" << endl;
		 while (!isExit)
		 {
			 if (!cam.read(frame))
			 {
				 msleep(1);
				 continue;
			 }
			 if (frame.empty())
			 {
				 msleep(1);
				 continue;
			 }
			 //确保数据是连续的
			 XData d((char *)frame.data, frame.cols*frame.rows*frame.elemSize(),GetCurTime());
			 Push(d);
		 }
		 cout << "退出视频线程" << endl;
	 }
	 bool Init(int camIdex = 0)
	 {
		 //使用opencv打开本地相机
		 cam.open(camIdex);
		 ///1.打开摄像头
		 if (!cam.isOpened())
		 {
			 cout<<"cam open failed!";
			 return false;
		 }
		 cout << "cam open success" << endl;
		 width = cam.get(CAP_PROP_FRAME_WIDTH);
		 height = cam.get(CAP_PROP_FRAME_HEIGHT);
		 fps = cam.get(CAP_PROP_FPS);
		 if (fps == 0)fps = 25;
		 return true;
	 }
	 bool Init(const char *url)
	 {
		 //使用opencv打开本地相机
		 cam.open(url);
		 ///1.打开摄像头
		 if (!cam.isOpened())
		 {
			 cout << "cam open failed!";
			 return false;
		 }
		 cout << "cam open success" << endl;
		 width = cam.get(CAP_PROP_FRAME_WIDTH);
		 height = cam.get(CAP_PROP_FRAME_HEIGHT);
		 fps = cam.get(CAP_PROP_FPS);
		 if (fps == 0)fps = 25;
		 return true;
	 }
	 void Stop()
	 {
		 XDataThread::Stop();
		 if (cam.isOpened())
		 {
			 cam.release();
		 }
	 }
};
XVideoCapture *XVideoCapture::Get(unsigned char index)
{
	static CXVideoCapture xc[255];
	return &xc[index];
}
XVideoCapture::XVideoCapture()
{
}


XVideoCapture::~XVideoCapture()
{
}