利用命令行可以轻松实现录屏功能,网上的例子也有很多,这里就不再赘述。
之前有利用FFMPEG封装过摄像头采集的图像,也封装过音频设备采集的数据,录屏其实只需要更换音视频的输入源即可。
在将封装后的音视频流写入文件时,我使用了两个线程,本来计划在各自的线程统计下一帧的时间戳,通过比较,将时间戳靠前的音频或视频帧写入文件。实现过程中,发现可以不做这一步处理,哪个线程抢占到输出上下文,正常写入即可。
主要的代码如下所示,其实和之前的封装代码有很多类似的地方,主要是将两个采集流程理清楚。
#include "cworkthread.h"
#include <qdebug.h>
#include"cresampleaudioframe.h"
#include "cscalevideoframe.h"
CWorkThread::CWorkThread(QObject* parent):QThread(parent)
{
}
CWorkThread::~CWorkThread()
{
disconnect();
stop();
quit();
wait();
}
void CWorkThread::stop()
{
m_bRun = false;
if(pVideoThread->joinable())
{
pVideoThread->join();
}
if(pAudioThread->joinable())
{
pAudioThread->join();
}
if(false == isInterruptionRequested())
requestInterruption();
}
#define PRINT_ERROR(ret)\
if (ret < 0) {\
char buf[100]={0};\
av_strerror(ret,buf,100);\
qDebug()<<__FUNCTION__<<__LINE__<<"error code"<<ret<<" error info"<<buf;\
return ;\
}\
int createVideoInput(AVFormatContext*& pContext)
{
//获取输入格式对象
AVInputFormat *fmt = av_find_input_format("gdigrab");
//格式上下文 打开设备
pContext = avformat_alloc_context();
//设备名
//const char * deviceName = "audio=virtual-audio-capturer";
//const char * deviceName ="video=EasyCamera";// "video=ASUS USB2.0 WebCam";
const char * deviceName = "desktop";
AVDictionary *options = nullptr;
//options = (AVDictionary*)(av_malloc(sizeof(AVDictionary)));
av_dict_set(&options, "video_size", "1920*1080", 0);
// av_dict_set(&options, "offset_x", "1280", 0);
//av_dict_set(&options, "offset_y", "720", 0);
//av_dict_set(&options,"video_size","640*480",0);
//av_dict_set(&options,"pixel_format","yuyv422",0);
av_dict_set(&options,"framerate","10",0);
//av_dict_set_int(&options,"framerate",30,0);
int ret = avformat_open_input(&pContext,deviceName,fmt,&options);
return ret;
}
int createDecoder(AVCodecContext*& pDecoderCtx,AVCodecParameters* codecpar)
{
const AVCodec* pDecoder;
pDecoder = avcodec_find_decoder(codecpar->codec_id);
pDecoderCtx = avcodec_alloc_context3(pDecoder);
avcodec_parameters_to_context(pDecoderCtx,codecpar);
//打开解码器
int ret = avcodec_open2(pDecoderCtx,pDecoder,nullptr);
return ret;
}
int createVideoDecoder(AVCodecContext*& pDecoderCtx,AVStream* videoStream)
{
const AVCodec* pDecoder;
pDecoder = avcodec_find_decoder(videoStream->codecpar->codec_id);
pDecoderCtx = avcodec_alloc_context3(pDecoder);
avcodec_parameters_to_context(pDecoderCtx,videoStream->codecpar);
//打开解码器
int ret = avcodec_open2(pDecoderCtx,pDecoder,nullptr);
return ret;
}
int createVideoEncoder(AVCodecContext*& pEncoderCtx,AVStream* videoStream)
{
const AVCodec* pEncoder ;
AVCodecID codecId = AV_CODEC_ID_H264;
pEncoder = avcodec_find_encoder(codecId);
pEncoderCtx = avcodec_alloc_context3(pEncoder);
pEncoderCtx->codec_id = codecId;
//codecCtx->bit_rate = 400000;
pEncoderCtx->pix_fmt = *pEncoder->pix_fmts;// AV_PIX_FMT_YUV420P;
pEncoderCtx->width = videoStream->codecpar->width;
pEncoderCtx->height = videoStream->codecpar->height;
pEncoderCtx->time_base = videoStream->time_base;
//pEncoderCtx->framerate = videoStream->r_frame_rate;
pEncoderCtx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
int ret = avcodec_open2(pEncoderCtx,pEncoder,nullptr);
return ret;
}
void CWorkThread::initCommon()
{
avdevice_register_all();
pOutFormatCtx = nullptr;
pOutFormatCtx = avformat_alloc_context();
std::string outFilePath = "d:/capture.mp4";
AVOutputFormat* fmt = av_guess_format(NULL, outFilePath.c_str(), NULL);
avformat_alloc_output_context2(&pOutFormatCtx, fmt, fmt->name, outFilePath.c_str());
}
void CWorkThread::readyVideoCapture()
{
//打开摄像头设备
pVideoInputCtx=nullptr;
int ret = createVideoInput(pVideoInputCtx);
PRINT_ERROR(ret);
//查找视频流
int videoStreamId = av_find_best_stream(pVideoInputCtx, AVMEDIA_TYPE_VIDEO, -1, -1, NULL, NULL);
pVideoInputStream = pVideoInputCtx->streams[videoStreamId];
pVideoInputStream->codecpar->width = 1920;
pVideoInputStream->codecpar->height =1080 ;
//videoStream->time_base = ;
pVideoInputStream->r_frame_rate = AVRational{1000000,100000};
//int fps = av_q2d(pVideoInputStream->r_frame_rate);
//AVRational videoFrameRate{1,fps};
/*初始化解码器上下文*/
ret = createDecoder(pVideoDecoderCtx,pVideoInputStream->codecpar);
PRINT_ERROR(ret);
/*查找编码器,初始化编码器上下文*/
ret = createVideoEncoder(pVideoEncoderCtx,pVideoInputStream);
PRINT_ERROR(ret);
/*创建输出流*/
AVStream * outVideoStream;
outVideoStream = avformat_new_stream(pOutFormatCtx,pVideoEncoderCtx->codec);
avcodec_parameters_from_context(outVideoStream->codecpar,pVideoEncoderCtx);
outVideoStream->time_base = av_inv_q(pVideoInputStream->r_frame_rate);
}
void CWorkThread::readyAudioCapture()
{
//打开设备
pAudioInputCtx = nullptr;
std::string audioDevice = "audio=virtual-audio-capturer";//"audio=" + audioDevice;
AVInputFormat* AudioInputFormat = av_find_input_format("dshow");
AVDictionary* AudioOptions=nullptr;
int ret = avformat_open_input(&pAudioInputCtx, audioDevice.c_str(), AudioInputFormat, &AudioOptions);
//查找音频流
if(avformat_find_stream_info(pAudioInputCtx,NULL)< 0)
{
return;
}
//int audioStreamId = 0;
int StreamsNumber = (int)pAudioInputCtx->nb_streams;
pAudioInputStream = NULL;
for (int i = 0; i < StreamsNumber; i++) {
if (pAudioInputCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
audioStreamInputId = i;
pAudioInputStream = pAudioInputCtx->streams[i];
break;
}
}
AVCodecParameters *AudioParams = pAudioInputStream->codecpar;
createDecoder(pAudioDecoderCtx,AudioParams);
//创建音频输出流
AVStream *AudioStreamOut = avformat_new_stream(pOutFormatCtx, NULL);
/*查找编码器,初始化编码器上下文*/
AVCodec* AudioCodecOut = avcodec_find_encoder(AV_CODEC_ID_AAC);
pAudioEncoderCtx = avcodec_alloc_context3(AudioCodecOut);
if ((AudioCodecOut)->supported_samplerates) {
pAudioEncoderCtx->sample_rate = (AudioCodecOut)->supported_samplerates[0];
for (int i = 0; (AudioCodecOut)->supported_samplerates[i]; i++) {
if ((AudioCodecOut)->supported_samplerates[i] == pAudioDecoderCtx->sample_rate)
pAudioEncoderCtx->sample_rate = pAudioDecoderCtx->sample_rate;
}
}
pAudioEncoderCtx->codec_id = AV_CODEC_ID_AAC;
pAudioEncoderCtx->bit_rate = 128000;
pAudioEncoderCtx->channels = pAudioDecoderCtx->channels;
pAudioEncoderCtx->channel_layout = av_get_default_channel_layout(pAudioEncoderCtx->channels);
pAudioEncoderCtx->sample_fmt = AudioCodecOut->sample_fmts ? AudioCodecOut->sample_fmts[0] : AV_SAMPLE_FMT_FLTP;
pAudioEncoderCtx->time_base = {1, pAudioDecoderCtx->sample_rate};
pAudioEncoderCtx->strict_std_compliance = FF_COMPLIANCE_EXPERIMENTAL;
if (pOutFormatCtx->oformat->flags & AVFMT_GLOBALHEADER)
pAudioEncoderCtx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
avcodec_open2(pAudioEncoderCtx, AudioCodecOut, NULL);
// int audioIndexOut = 0;
for (int i = 0; i < (int)pOutFormatCtx->nb_streams; i++) {
if (pOutFormatCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_UNKNOWN) {
audioStreamOutId = i;
}
}
avcodec_parameters_from_context(pOutFormatCtx->streams[audioStreamOutId]->codecpar, pAudioEncoderCtx);
}
void CWorkThread::createOutFile()
{
const std::string outFilePath = "d:/capture.mp4";
// create an empty video file
if (!(pOutFormatCtx->flags & AVFMT_NOFILE)) {
if (avio_open2(&pOutFormatCtx->pb, outFilePath.c_str(), AVIO_FLAG_WRITE, NULL, NULL) < 0) {
return;
}
}
if (pOutFormatCtx->nb_streams == 0) {
return;
}
if (avformat_write_header(pOutFormatCtx, NULL) < 0) {
return;
}
}
void CWorkThread::captureAudio()
{
AVPacket* pRawAudioPkt = av_packet_alloc();
av_init_packet(pRawAudioPkt);
AVPacket* reEncodecdAudioPkt = av_packet_alloc();//存编码后的数据
av_init_packet(reEncodecdAudioPkt);
AVFrame* pRawAudioFrame= av_frame_alloc();
int sampleCount = 0;
CResampleAudioFrame cresampler;
cresampler.setTarget( pAudioEncoderCtx->sample_fmt, pAudioEncoderCtx->channel_layout, pAudioEncoderCtx->sample_rate);
while(m_bRun)
{
int res = av_read_frame(pAudioInputCtx,pRawAudioPkt);
if(res>=0 && pRawAudioPkt->stream_index == audioStreamInputId)
{
int ret = avcodec_send_packet(pAudioDecoderCtx, pRawAudioPkt);
if(ret < 0)
{
continue;
}
av_packet_unref(pRawAudioPkt);
while(m_bRun )
{
ret = avcodec_receive_frame(pAudioDecoderCtx, pRawAudioFrame);
if ( ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
{
break;
}
AVFrame* pResampledFrame = nullptr;
cresampler.pushSourceFrame(pRawAudioFrame);
while((pResampledFrame = cresampler.getReampledFrame(pAudioEncoderCtx->frame_size)) !=nullptr)
{
pResampledFrame->pts = sampleCount;
sampleCount += pResampledFrame->nb_samples;
avcodec_send_frame(pAudioEncoderCtx, pResampledFrame);
while (ret >= 0)
{
ret = avcodec_receive_packet(pAudioEncoderCtx, reEncodecdAudioPkt);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
break;
av_packet_rescale_ts(reEncodecdAudioPkt, pAudioEncoderCtx->time_base, pOutFormatCtx->streams[audioStreamOutId]->time_base);
reEncodecdAudioPkt->stream_index = audioStreamOutId;
outFormatCtxMtx.lock();
av_write_frame(pOutFormatCtx, reEncodecdAudioPkt);
outFormatCtxMtx.unlock();
av_packet_unref(reEncodecdAudioPkt);
}
}
}
}
}
av_frame_free(&pRawAudioFrame);
av_packet_free(&reEncodecdAudioPkt);
av_packet_free(&pRawAudioPkt);
avformat_close_input(&pAudioInputCtx);
qDebug()<<"end of audio capture";
}
void CWorkThread::captureVideo()
{
//打开摄像头设备
AVFormatContext* pContext = pVideoInputCtx;
//查找视频流
AVStream* videoStream = pVideoInputStream;
int fps = av_q2d(videoStream->r_frame_rate);
AVRational targetVideoTimeBase{1,fps};
AVRational srcVideoTimeBase = videoStream->time_base;
AVPacket* reEncodecdPkt = av_packet_alloc();//存编码后的数据
int videoFrameCount =0; //解码视频帧计数
AVPacket* pVideoPkt = av_packet_alloc(); //解码前视频压缩包
AVFrame* pVideoFrame = av_frame_alloc(); //解码后视频帧
CScaleVideoFrame frameScaler;
frameScaler.setTarget(1920,1080,AV_PIX_FMT_YUV420P);//videoStream->codecpar->width,videoStream->codecpar->height,AV_PIX_FMT_RGB24);// pEncoderCtx->pix_fmt);
while(m_bRun)
{
int res = av_read_frame(pContext,pVideoPkt);
if(res == 0)
{
//解码
if(pVideoPkt->stream_index != AVMEDIA_TYPE_VIDEO)
{
continue;
}
int ret = avcodec_send_packet(pVideoDecoderCtx, pVideoPkt);
if(ret <0)
{
break;
}
av_packet_unref(pVideoPkt);
while(m_bRun)
{
ret = avcodec_receive_frame(pVideoDecoderCtx, pVideoFrame);
if ( ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
{
break;
}
//转换像素格式
AVFrame* scaled_frame = frameScaler.getScaledFrame(pVideoFrame);
if(scaled_frame == NULL)
{
break;
}
//continue;
//根据帧率和时间基计算pts
int64_t pts = av_rescale_q(videoFrameCount, targetVideoTimeBase, srcVideoTimeBase);
scaled_frame->pts = pts;
++videoFrameCount;
//编码
ret = avcodec_send_frame(pVideoEncoderCtx, scaled_frame);
ret = avcodec_receive_packet(pVideoEncoderCtx, reEncodecdPkt);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
{
break;
}
//写入文件
outFormatCtxMtx.lock();
ret = av_interleaved_write_frame(pOutFormatCtx, reEncodecdPkt);
outFormatCtxMtx.unlock();
av_packet_unref(reEncodecdPkt);
if(ret <0)
{
break;
}
}
}
else if(res == AVERROR(ERANGE))
{
continue;
}
else
{
break;
}
}
av_packet_free(&reEncodecdPkt);
av_frame_free(&pVideoFrame);
avcodec_free_context(&pVideoEncoderCtx);
avcodec_free_context(&pVideoDecoderCtx);
avformat_close_input(&pContext);
av_packet_unref(pVideoPkt);
qDebug()<<"end of video capture,frame count"<<videoFrameCount;
}
void CWorkThread::run()
{
m_bRun = true;
avdevice_register_all();
initCommon();
readyVideoCapture();
readyAudioCapture();
createOutFile();
pVideoThread =new std::thread([=](){
captureVideo();
});
pAudioThread =new std::thread([=](){
captureAudio();
});
while(false == isInterruptionRequested())
{
std::this_thread::sleep_for(std::chrono::milliseconds(1000));
}
av_write_trailer(pOutFormatCtx);
avio_close(pOutFormatCtx->pb);
avformat_free_context(pOutFormatCtx);
qDebug()<<"end of work thread";
delete pAudioThread;
delete pVideoThread;
}
工作线程的头文件定义如下
#ifndef CWORKTHREAD_H
#define CWORKTHREAD_H
#include<QThread>
#include <mutex>
#include <thread>
class AVFormatContext;
class AVCodecContext;
class AVStream;
class CWorkThread:public QThread
{
public:
CWorkThread(QObject* parent = NULL);
~CWorkThread();
//void start();
void stop();
private:
void run() override;
void initCommon();
void readyVideoCapture();
void readyAudioCapture();
void createOutFile();
void captureAudio();
void captureVideo();
bool m_bRun;
AVFormatContext* pOutFormatCtx=nullptr;
std::mutex outFormatCtxMtx;
//video params
AVFormatContext* pVideoInputCtx;
AVCodecContext* pVideoDecoderCtx;
AVCodecContext* pVideoEncoderCtx;
AVStream * pVideoInputStream;
//audio params
AVFormatContext* pAudioInputCtx;
AVCodecContext* pAudioDecoderCtx;
AVCodecContext* pAudioEncoderCtx;
AVStream * pAudioInputStream;
int audioStreamInputId = 0;
int audioStreamOutId=0;
std::thread* pAudioThread;
std::thread* pVideoThread;
};
#endif // CWORKTHREAD_H
关于音频重采样部分的代码,我做了简单的封装,这部分的功能相对独立,提出来方便理解整体思路。
#ifndef CRESAMPLEAUDIOFRAME_H
#define CRESAMPLEAUDIOFRAME_H
extern "C"{
#include <libavdevice/avdevice.h>
#include <libavformat/avformat.h>
#include <libavutil/avutil.h>
#include <libavcodec/avcodec.h>
#include <libavutil/imgutils.h>
#include <libswscale/swscale.h>
#include <libswresample/swresample.h>
#include "libavutil/audio_fifo.h"
}
/**
* @brief 重采样类
* 设置输出的采样格式、采样率和通道布局,传入AVFrame,获取重采样后的数据
*/
class CResampleAudioFrame
{
public:
CResampleAudioFrame();
~CResampleAudioFrame();
void setTarget(AVSampleFormat sampleFmt,uint64_t chanLayout,uint64_t sampleRate);
/**
* @brief 设置需要重采样的frame
* @param srcFrame 不能为空
*
* @return
*/
int pushSourceFrame(AVFrame* srcFrame);
AVFrame* getReampledFrame(int sampleCnt);
private:
void initSwr();
private:
SwrContext* m_swrCtx=nullptr;
AVSampleFormat m_srcFmt; /**重采样前格式*/
uint64_t m_srcChanLayout; /**重采样前布局*/
int m_srcSampleRate; /**重采样前采样率*/
//uint8_t* m_pcmBuf; /**存放解码后的pcm*/
int32_t m_pcmLen; /**存放解码后的pcm缓冲区长度*/
AVSampleFormat m_fmt; /**重采样目的格式*/
uint64_t m_chanLayout; /**重采样目的布局*/
uint64_t m_sampleRate; /**重采样目的采样率*/
int m_channels;
AVFrame* m_audioSwrFrame;
int dstSamples;
uint8_t **resampledData;
int nb_samples;
AVAudioFifo *AudioFifoBuff;
};
#endif // CRESAMPLEAUDIOFRAME_H
#include "cresampleaudioframe.h"
#include "QDebug"
CResampleAudioFrame::CResampleAudioFrame()
{
m_sampleRate = 44100;
m_fmt = AV_SAMPLE_FMT_S16;
m_chanLayout = AV_CH_LAYOUT_STEREO;
m_channels = av_get_channel_layout_nb_channels(AV_CH_LAYOUT_STEREO);
m_srcFmt=AVSampleFormat::AV_SAMPLE_FMT_NONE; /**重采样前格式*/
m_srcChanLayout = 0; /**重采样前布局*/
m_srcSampleRate = 0; /**重采样前采样率*/
}
CResampleAudioFrame::~CResampleAudioFrame()
{
if(m_swrCtx)
{
swr_free(&m_swrCtx);
m_swrCtx = nullptr;
av_frame_free(&m_audioSwrFrame);
av_audio_fifo_free(AudioFifoBuff);
if(resampledData)
{
av_freep(&resampledData[0]);
}
free(resampledData);
}
}
void CResampleAudioFrame::setTarget(AVSampleFormat sampleFmt, uint64_t chanLayout, uint64_t sampleRate)
{
bool bChanged = false;
if(m_sampleRate != sampleRate ||
m_chanLayout != chanLayout ||
m_fmt != sampleFmt)
{
bChanged = true;
}
m_sampleRate = sampleRate;
m_chanLayout = chanLayout;
m_fmt = sampleFmt;
m_channels = av_get_channel_layout_nb_channels(chanLayout);
if(bChanged&&
m_srcFmt!=AV_SAMPLE_FMT_NONE &&
m_srcChanLayout!=0 &&
m_srcSampleRate!=0 &&
m_swrCtx)
{//目标发生改变,并且已经创建了上下文,需要重新生成
initSwr();
}
}
int CResampleAudioFrame::pushSourceFrame(AVFrame *srcFrame)
{
if(srcFrame == nullptr)
return -1;
//输出缓冲区样本数量
dstSamples=av_rescale_rnd(srcFrame->nb_samples,
m_sampleRate,
srcFrame->sample_rate,
AVRounding::AV_ROUND_UP);
int32_t bytesPerSample = av_get_bytes_per_sample((AVSampleFormat) m_fmt);
if(srcFrame->channel_layout == 0)
srcFrame->channel_layout = av_get_default_channel_layout(srcFrame->channels);
if(
m_srcChanLayout != srcFrame->channel_layout ||
m_srcFmt != (AVSampleFormat) srcFrame->format ||
m_srcSampleRate != srcFrame->sample_rate)
{
m_srcChanLayout= srcFrame->channel_layout;
m_srcFmt = (AVSampleFormat) srcFrame->format;
m_srcSampleRate = srcFrame->sample_rate;
nb_samples = srcFrame->nb_samples;
initSwr();
}
{
resampledData = (uint8_t **)calloc(m_channels, sizeof(*resampledData));
int alloced = av_samples_alloc(resampledData, nullptr, m_channels,
srcFrame->nb_samples, m_fmt, 0);
if(alloced<0)
{
return -1;
}
}
//resamp
int res = swr_convert(m_swrCtx,
resampledData,dstSamples,
(const uint8_t**)srcFrame->extended_data,srcFrame->nb_samples);
m_audioSwrFrame->channel_layout = m_chanLayout;
m_audioSwrFrame->sample_rate = m_sampleRate ;
m_audioSwrFrame->channels = m_channels;
m_audioSwrFrame->format = m_fmt;
m_audioSwrFrame->nb_samples = res;
int bufsieze = av_audio_fifo_size(AudioFifoBuff);
int ret = av_audio_fifo_realloc(AudioFifoBuff, bufsieze + res);
ret = av_audio_fifo_write(AudioFifoBuff, (void **)resampledData, res);
return ret;
}
AVFrame *CResampleAudioFrame::getReampledFrame(int sampleCnt)
{
int fifoSize =av_audio_fifo_size(AudioFifoBuff);
if (fifoSize < sampleCnt)
{
return nullptr;
}
int res = av_audio_fifo_read(AudioFifoBuff, (void **)(m_audioSwrFrame->data), sampleCnt);
m_audioSwrFrame->nb_samples= res;
if( res >= 0)
{
return m_audioSwrFrame;
}
return nullptr;
}
void CResampleAudioFrame::initSwr()
{
if(m_swrCtx)
{
swr_free(&m_swrCtx);
m_swrCtx = nullptr;
av_frame_free(&m_audioSwrFrame);
}
if(m_swrCtx == nullptr)
{
m_swrCtx = swr_alloc_set_opts(nullptr,
m_chanLayout,m_fmt,m_sampleRate,
m_srcChanLayout,
m_srcFmt,
m_srcSampleRate,
0,nullptr);
m_audioSwrFrame = av_frame_alloc();
m_audioSwrFrame->sample_rate = m_sampleRate;
m_audioSwrFrame->format = m_fmt;
m_audioSwrFrame->nb_samples = 1024;
m_audioSwrFrame->channel_layout = (uint64_t)m_chanLayout;
av_frame_get_buffer(m_audioSwrFrame, 0); // 分配缓冲区
swr_init(m_swrCtx);
AudioFifoBuff = av_audio_fifo_alloc(m_fmt, m_channels, 1);
}
}