FFmpeg解码提速之libyuv

1. FFmpeg解码瓶颈

经测试发现，FFmpeg解码瓶颈在YUV转RGB上，在12MP视频的环境下，单帧转换时间超过40ms，效率无法满足要求
FFmpeg的YUV转RGB代码：

sws_scale(img_convert_ctx, static_cast<uint8_t const * const *> (pFrame->data),
			pFrame->linesize, 0, pCodecCtx->height, dst_data, dst_linesize);

2. 使用libyuv提升解码效率

调用libyuv库函数，将yuv转换成RGB，可以明显改善解码效率
代码：

libyuv::I420ToABGR(
	pFrame->data[0], pFrame->linesize[0],
	pFrame->data[2], pFrame->linesize[2],
	pFrame->data[1], pFrame->linesize[1],
	pFrameRGB->data[0], pFrameRGB->linesize[0],
	pCodecCtx->width, pCodecCtx->height);

3. 完整代码

#include "FFmpegRtsp.h"

extern "C"
{
#include "libavcodec/avcodec.h"
#include "libavformat/avformat.h"
#include "libavutil/pixfmt.h"
#include "libswscale/swscale.h"
#include "libavutil/time.h"
#include "libavutil/mathematics.h"
#include "libavutil/imgutils.h"
}
#include "libyuv.h"

void FFmpegRtsp::readVideoFrame()
{
	AVFormatContext* pFormatCtx;
	AVCodecContext* pCodecCtx;
	AVCodec* pCodec;
	AVFrame* pFrame;
	AVFrame* pFrameRGB;
	AVPacket* pPacket;

	//static struct SwsContext *img_convert_ctx;
	int videoStream = -1;
	int ret;

	avformat_network_init();			  //初始化FFmpeg网络模块		   
	pFormatCtx = avformat_alloc_context();//Allocate an AVFormatContext.

	AVDictionary* avdic = nullptr;
	char option_key[] = "rtsp_transport";
	char option_value[] = "tcp";
	av_dict_set(&avdic, option_key, option_value, 0);
	char option_key2[] = "max_delay";
	char option_value2[] = "2000";
	av_dict_set(&avdic, option_key2, option_value2, 0);
	//打开流
	if (avformat_open_input(&pFormatCtx, _url.c_str(), nullptr, &avdic) != 0) {
		printf("can't open input url = %s\n",_url.c_str());
		return;
	}

	//循环查找视频中包含的流信息，直到找到视频类型的流
	//便将其记录下来 保存到videoStream变量中
	//这里只处理视频流  音频流先不管他
	if (avformat_find_stream_info(pFormatCtx, nullptr) < 0) {
		printf("Could't find stream infomation.\n");
		return;
	}
	for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++) {
		if (pFormatCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
			videoStream = static_cast<int>(i);
		}
	}
	if (videoStream == -1) {
		printf("Didn't find a video stream.\n");
		return;
	}

	//查找解码器
	pCodec = avcodec_find_decoder(pFormatCtx->streams[videoStream]->codecpar->codec_id);
	pCodecCtx = avcodec_alloc_context3(pCodec);
	avcodec_parameters_to_context(pCodecCtx, pFormatCtx->streams[videoStream]->codecpar);
	pCodecCtx->bit_rate = 0;   //初始化为0
	pCodecCtx->time_base.num = 1;  //下面两行：一秒钟25帧
	pCodecCtx->time_base.den = 10;
	pCodecCtx->frame_number = 1;  //每包一个视频帧

	if (pCodec == nullptr) {
		printf("Codec not found.\n");
		return;
	}

	//打开解码器
	if (avcodec_open2(pCodecCtx, pCodec, nullptr) < 0) {
		printf("Could not open codec.\n");
		return;
	}

	pFrame = av_frame_alloc();
	pFrameRGB = av_frame_alloc();

	int numBytes = av_image_get_buffer_size(AV_PIX_FMT_BGRA, pCodecCtx->width, pCodecCtx->height, 1);
	uint8_t* out_buffer = static_cast<uint8_t*>(av_malloc(static_cast<size_t>(static_cast<uint64_t>(numBytes) * sizeof(uint8_t))));
	uint8_t* dst_data[4];
	int dst_linesize[4];
	av_image_fill_arrays(dst_data, dst_linesize, out_buffer, AV_PIX_FMT_BGRA, pCodecCtx->width, pCodecCtx->height, 1);

	//分配一个packet
	int y_size = pCodecCtx->width * pCodecCtx->height;
	pPacket = static_cast<AVPacket*>(malloc(sizeof(AVPacket)));
	av_new_packet(pPacket, y_size); //分配packet的数据

	//cv::namedWindow("frame", cv::WINDOW_KEEPRATIO);
#define FFMPEG 0
#if FFMPEG
	printf("ffmpeg convert ing...\n");
    // R9 数据格式转换上下文
    struct SwsContext *img_convert_ctx = sws_getContext(
            pCodecCtx->width, pCodecCtx->height, pCodecCtx->pix_fmt,
            pCodecCtx->width, pCodecCtx->height, AV_PIX_FMT_BGRA,
            SWS_BICUBIC, NULL, NULL, NULL);
#endif
	while (m_bStop == false)
	{
		if (av_read_frame(pFormatCtx, pPacket) < 0)
		{
			break; //这里认为视频读取完了
		}

		if (pPacket->stream_index == videoStream)
		{
			auto start = std::chrono::system_clock::now();	
			avcodec_send_packet(pCodecCtx, pPacket);
			ret = avcodec_receive_frame(pCodecCtx, pFrame);
			if (ret < 0)
			{
				printf("receive frame from packet error.\n");
				continue;
			}
#if FFMPEG
			//YUV-->RGB ffmpeg的转换方法也行，但是在12MP视频的环境下，单帧转换时间超过40ms，效率无法满足要求
			sws_scale(img_convert_ctx, static_cast<uint8_t const * const *> (pFrame->data),
			pFrame->linesize, 0, pCodecCtx->height, dst_data, dst_linesize);

			cv::Mat frame = cv::Mat(cv::Size(pCodecCtx->width, pCodecCtx->height), CV_8UC4);
			frame.data = dst_data[0];
#else
			//fix buffer//调用libyuv库函数，将yuv转换成RGB，效率好过ffmpeg几倍
			avpicture_fill(
				(AVPicture*)pFrameRGB,                       //rgb帧结构地址，但只有data的首地址
				out_buffer,                                   //rgb帧结构data 数据存放空间，提前分配好，分配一次即可，循环覆盖
				AV_PIX_FMT_BGRA,
				pCodecCtx->width,
				pCodecCtx->height);

			libyuv::I420ToABGR(
				pFrame->data[0], pFrame->linesize[0],
				pFrame->data[2], pFrame->linesize[2],
				pFrame->data[1], pFrame->linesize[1],
				pFrameRGB->data[0], pFrameRGB->linesize[0],
				pCodecCtx->width, pCodecCtx->height);

			cv::Mat frame = cv::Mat(cv::Size(pCodecCtx->width, pCodecCtx->height), CV_8UC4);
			frame.data = pFrameRGB->data[0];
#endif
			
			std::unique_lock<std::mutex> lk(_lock);
			if (_frameQueue.size() > _frameLen)
			{				
				cv::Mat tmp = _frameQueue.front();
				tmp.release();
				_frameQueue.pop();
			}

			_frameQueue.push(frame);
			_cond.notify_one();
			lk.unlock();
		}
		av_packet_unref(pPacket);
	}

	av_free(out_buffer);
	av_free(pFrameRGB);
	av_free(pFrame);
	av_packet_free(&pPacket);
	avcodec_close(pCodecCtx);
	avformat_close_input(&pFormatCtx);
}

=提示：上述代码是用来rtsp拉流，然后转Mat给opencv使用的=