文章目录
本项目采用生产者消费者模型,生产者线程:使用ffmpeg将mp4格式数据解析为yuv的帧,消费者线程:利用sdl2将解析的yuv的帧进行消费,绘制到屏幕上。未完成的部分:1.解析音频数据,并与视频数据同步。2.增加界面:暂停,播放按钮,支持视频前进和后退。
学习音视频的参考资料与项目:
playdemo_github
雷神的csdn博客
1.FFMPEG利用命令行将mp4转yuv420
ffmpeg -i input.mp4 -c:v rawvideo -pix_fmt yuv420p output.yuv
2.ffmpeg将mp4解析为yuv数据
2.1 核心api:
- av_read_frame:读取一帧数据
- avcodec_send_packet:将数据包发送给解码器
- avcodec_receive_frame:将数据包从解码器中取
- sws_scale:格式转换,将解码后的帧数据转为yuv数据,存储在data[0],data[1],data[2]中
void readFrame()
{
AVPacket* avPacket = av_packet_alloc();
AVFrame* frame = av_frame_alloc();
FILE* fp = fopen("F:/VS_Project/ffmpeg_demo/yuv.data","wb+") ;
while (av_read_frame(formatContext, avPacket) >= 0 && fp)
{
if (avPacket->stream_index == videoStreamIndex)
{
if (avcodec_send_packet(codecContext, avPacket) < 0) {
std::cerr << "发送数据包到解码器失败" << std::endl;
break;
}
/*解码*/
int ret = avcodec_receive_frame(codecContext, frame);
printf("ret:%d\n", ret);
if (ret >= 0)
{
ret = sws_scale(swsContext, frame->data, frame->linesize, 0, codecContext->height, yuvFrame->data, yuvFrame->linesize);
printf("sws_scale ret=%d\n", ret);
std::lock_guard<std::mutex>lck(mtx);
isFinished = false;
memcpy(yuvBuf, yuvFrame->data[0], yuvFrame->width * yuvFrame->height);
memcpy(yuvBuf + yuvFrame->width * yuvFrame->height, yuvFrame->data[1], yuvFrame->width * yuvFrame->height / 4);
memcpy(yuvBuf + yuvFrame->width * yuvFrame->height*5/4, yuvFrame->data[2], yuvFrame->width * yuvFrame->height / 4);
isFinished = true;
condvar.notify_one();
//保存y分量
//fwrite(yuvFrame->data[0], 1, yuvFrame->width * yuvFrame->height, fp);
//保存uv分量
//fwrite(yuvFrame->data[1], 1, yuvFrame->width * yuvFrame->height/4, fp);
//fwrite(yuvFrame->data[2], 1, yuvFrame->width * yuvFrame->height / 4, fp);
}
}
}
fclose(fp);
av_frame_unref(yuvFrame);
av_packet_free(&avPacket);
av_frame_unref(frame);
}
3.SDL2进行yuv绘制到屏幕
3.1 核心api
- SDL_Init
- SDL_CreateWindow
- SDL_CreateRenderer
- SDL_CreateTexture
- SDL_UpdateTexture
- SDL_RenderCopy
- SDL_RenderPresent
- SDL_Delay:控制帧率
int sdl_display()
{
if (SDL_Init(SDL_INIT_VIDEO)) {
printf("sdl init failed\n");
return -1;
}
SDL_Window* window = SDL_CreateWindow("sdl_demo", 200, 200, codecContext->width, codecContext->height, SDL_WINDOW_OPENGL | SDL_WINDOW_RESIZABLE);
if (!window) {
SDL_Quit();
return -1;
}
SDL_Renderer* renderer = SDL_CreateRenderer(window, -1, 0);
if (!renderer)
{
SDL_DestroyWindow(window);
SDL_Quit();
return -1;
}
SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);
SDL_RenderClear(renderer);
Uint32 pixformat = SDL_PIXELFORMAT_IYUV;
SDL_Texture* sdlTexture = SDL_CreateTexture(renderer, pixformat, SDL_TEXTUREACCESS_STREAMING, codecContext->width, codecContext->height);
//FILE* fp = fopen("F:/VS_Project/ffmpeg_demo/yuv.data", "rb+");
while (1) {
//int ret = fread(yuvBuf, 1, yuvlen, fp);
//if (ret <= 0) {
// break;
//}
std::unique_lock<std::mutex>lck(mtx);
if (condvar.wait_for(lck, std::chrono::seconds(1), [] {
return isFinished;}))
{
isFinished = false;
SDL_UpdateTexture(sdlTexture, NULL, yuvBuf, codecContext->width);
SDL_RenderCopy(renderer, sdlTexture, NULL, NULL);
SDL_RenderPresent(renderer);
//控制帧率25fps
SDL_Delay(40);
}
else {
printf("sdl thread exit!\n");
break;
}
}
SDL_Quit();
return 0;
}
4.完整代码
-使用两个线程,生产者消费者模型
#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <stdexcept>
#include <iostream>
#include <string>
#include <thread>
#include <fstream>
#include <mutex>
#include <condition_variable>
extern "C" {
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/avutil.h>
#include <libswscale/swscale.h>
#include <SDL.h>
}
#undef main
#pragma warning(disable:4996)
AVFormatContext* formatContext = nullptr;
AVCodecContext* codecContext = nullptr;
SwsContext* swsContext = nullptr;
int videoStreamIndex = -1;
AVFrame* yuvFrame;
unsigned char* yuvBuf;
bool isReady = false;
bool isFinished = false;
std::mutex mtx;
std::condition_variable condvar;
void readFrame()
{
AVPacket* avPacket = av_packet_alloc();
AVFrame* frame = av_frame_alloc();
FILE* fp = fopen("F:/VS_Project/ffmpeg_demo/yuv.data","wb+") ;
while (av_read_frame(formatContext, avPacket) >= 0 && fp)
{
if (avPacket->stream_index == videoStreamIndex)
{
if (avcodec_send_packet(codecContext, avPacket) < 0) {
std::cerr << "发送数据包到解码器失败" << std::endl;
break;
}
/*解码*/
int ret = avcodec_receive_frame(codecContext, frame);
printf("ret:%d\n", ret);
if (ret >= 0)
{
ret = sws_scale(swsContext, frame->data, frame->linesize, 0, codecContext->height, yuvFrame->data, yuvFrame->linesize);
printf("sws_scale ret=%d\n", ret);
std::lock_guard<std::mutex>lck(mtx);
isFinished = false;
memcpy(yuvBuf, yuvFrame->data[0], yuvFrame->width * yuvFrame->height);
memcpy(yuvBuf + yuvFrame->width * yuvFrame->height, yuvFrame->data[1], yuvFrame->width * yuvFrame->height / 4);
memcpy(yuvBuf + yuvFrame->width * yuvFrame->height*5/4, yuvFrame->data[2], yuvFrame->width * yuvFrame->height / 4);
isFinished = true;
condvar.notify_one();
//保存y分量
//fwrite(yuvFrame->data[0], 1, yuvFrame->width * yuvFrame->height, fp);
//保存uv分量
//fwrite(yuvFrame->data[1], 1, yuvFrame->width * yuvFrame->height/4, fp);
//fwrite(yuvFrame->data[2], 1, yuvFrame->width * yuvFrame->height / 4, fp);
}
}
}
fclose(fp);
av_frame_unref(yuvFrame);
av_packet_free(&avPacket);
av_frame_unref(frame);
}
int sdl_display()
{
if (SDL_Init(SDL_INIT_VIDEO)) {
printf("sdl init failed\n");
return -1;
}
SDL_Window* window = SDL_CreateWindow("sdl_demo", 200, 200, codecContext->width, codecContext->height, SDL_WINDOW_OPENGL | SDL_WINDOW_RESIZABLE);
if (!window) {
SDL_Quit();
return -1;
}
SDL_Renderer* renderer = SDL_CreateRenderer(window, -1, 0);
if (!renderer)
{
SDL_DestroyWindow(window);
SDL_Quit();
return -1;
}
SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);
SDL_RenderClear(renderer);
Uint32 pixformat = SDL_PIXELFORMAT_IYUV;
SDL_Texture* sdlTexture = SDL_CreateTexture(renderer, pixformat, SDL_TEXTUREACCESS_STREAMING, codecContext->width, codecContext->height);
//FILE* fp = fopen("F:/VS_Project/ffmpeg_demo/yuv.data", "rb+");
while (1) {
//int ret = fread(yuvBuf, 1, yuvlen, fp);
//if (ret <= 0) {
// break;
//}
std::unique_lock<std::mutex>lck(mtx);
if (condvar.wait_for(lck, std::chrono::seconds(1), [] {
return isFinished;}))
{
isFinished = false;
SDL_UpdateTexture(sdlTexture, NULL, yuvBuf, codecContext->width);
SDL_RenderCopy(renderer, sdlTexture, NULL, NULL);
SDL_RenderPresent(renderer);
//控制帧率25fps
SDL_Delay(40);
}
else {
printf("sdl thread exit!\n");
break;
}
}
SDL_Quit();
return 0;
}
/*ffmpeg -i input.mp4 -c:v rawvideo -pix_fmt yuv420p output.yuv*/
int main(int argc, char* argv[])
{
/*if (argc != 2) {
std::cerr << "文件名未指定" << std::endl;
return -1;
}*/
std::string filename = "F:/VS_Project/ffmpeg_demo/1.mkv";
if (avformat_open_input(&formatContext, filename.c_str(), nullptr, nullptr) != 0)
{
std::cerr << "无法打开文件" << std::endl;
return -1;
}
if (avformat_find_stream_info(formatContext, nullptr) < 0) {
std::cerr << "无法找到视频流" << std::endl;
return -1;
}
for (int i = 0; i < formatContext->nb_streams; i++)
{
enum AVMediaType type = AVMEDIA_TYPE_VIDEO;
AVStream* st = formatContext->streams[i];
AVCodecParameters* codecpar = st->codecpar;
if (codecpar->codec_type == type)
{
videoStreamIndex = i;
const AVCodec* codec = avcodec_find_decoder(codecpar->codec_id);
codecContext = avcodec_alloc_context3(codec);
avcodec_parameters_to_context(codecContext, codecpar);
avcodec_open2(codecContext, codec, nullptr);
swsContext = sws_getContext(codecContext->width, codecContext->height, codecContext->pix_fmt,
codecContext->width, codecContext->height, AV_PIX_FMT_YUV420P,
SWS_BILINEAR, nullptr, nullptr, nullptr);
std::cout << "w:" << codecpar->width << std::endl;
std::cout << "h:" << codecpar->height << std::endl;
}
}
yuvFrame = av_frame_alloc();
yuvFrame->width = codecContext->width;
yuvFrame->height = codecContext->height;
yuvFrame->format = AV_PIX_FMT_YUV420P;
int yuvlen = codecContext->width * codecContext->height * 3 / 2;
yuvBuf = new unsigned char[yuvlen];
int ret = av_frame_get_buffer(yuvFrame, 0);
if (ret < 0) {
printf("分配缓冲区失败\n");
return -1;
}
//sdl_init();
std::thread th1(readFrame);
std::thread th2(sdl_display);
th1.join();
th2.join();
delete[]yuvBuf;
return 0;
}
5.效果展示
6.SDL2事件响应补充
6.1 处理方式-01
起一个refresh_video线程,用于产生一个自定义更新video的事件:REFRESH_EVENT,并且每40ms更新一次。while循环中持续等待事件的到来:SDL_WaitEvent,当收到事件后更新一帧画面。同时当监测到窗口改变的事件,SDL_GetWindowSize对窗口进行调整。
int refresh_video(void *opaque){
thread_exit=0;
while (thread_exit==0) {
SDL_Event event;
event.type = REFRESH_EVENT;
SDL_PushEvent(&event);
SDL_Delay(40);
}
thread_exit=0;
//Break
SDL_Event event;
event.type = BREAK_EVENT;
SDL_PushEvent(&event);
return 0;
}
SDL_Thread *refresh_thread = SDL_CreateThread(refresh_video,NULL,NULL);
SDL_Event event;
while(1){
//Wait
SDL_WaitEvent(&event);
if(event.type==REFRESH_EVENT){
if (fread(buffer, 1, pixel_w*pixel_h*bpp/8, fp) != pixel_w*pixel_h*bpp/8){
// Loop
fseek(fp, 0, SEEK_SET);
fread(buffer, 1, pixel_w*pixel_h*bpp/8, fp);
}
SDL_UpdateTexture( sdlTexture, NULL, buffer, pixel_w);
//FIX: If window is resize
sdlRect.x = 0;
sdlRect.y = 0;
sdlRect.w = screen_w;
sdlRect.h = screen_h;
SDL_RenderClear( sdlRenderer );
SDL_RenderCopy( sdlRenderer, sdlTexture, NULL, &sdlRect);
SDL_RenderPresent( sdlRenderer );
}else if(event.type==SDL_WINDOWEVENT){
//If Resize
SDL_GetWindowSize(screen,&screen_w,&screen_h);
}else if(event.type==SDL_QUIT){
thread_exit=1;
}else if(event.type==BREAK_EVENT){
break;
}
}
SDL_Quit();
6.2 处理方式-02
起一个事件循环线程,SDL_PeepEvents 从事件队列中取出一个事件,然后更新事件队列,并进行绘制操作。
//播放控制循环
void VideoCtl::LoopThread(VideoState *cur_stream)
{
SDL_Event event;
double incr, pos, frac;
m_bPlayLoop = true;
while (m_bPlayLoop)
{
double x;
refresh_loop_wait_event(cur_stream, &event);
switch (event.type) {
case SDL_KEYDOWN:
switch (event.key.keysym.sym) {
case SDLK_s: // S: Step to next frame
step_to_next_frame(cur_stream);
break;
case SDLK_a:
stream_cycle_channel(cur_stream, AVMEDIA_TYPE_AUDIO);
break;
case SDLK_v:
stream_cycle_channel(cur_stream, AVMEDIA_TYPE_VIDEO);
break;
case SDLK_c:
stream_cycle_channel(cur_stream, AVMEDIA_TYPE_VIDEO);
stream_cycle_channel(cur_stream, AVMEDIA_TYPE_AUDIO);
stream_cycle_channel(cur_stream, AVMEDIA_TYPE_SUBTITLE);
break;
case SDLK_t:
stream_cycle_channel(cur_stream, AVMEDIA_TYPE_SUBTITLE);
break;
default:
break;
}
break;
case SDL_WINDOWEVENT:
//窗口大小改变事件
qDebug()<<"SDL_WINDOWEVENT "<<endl;
switch (event.window.event) {
case SDL_WINDOWEVENT_RESIZED:
screen_width = cur_stream->width = event.window.data1;
screen_height = cur_stream->height = event.window.data2;
case SDL_WINDOWEVENT_EXPOSED:
cur_stream->force_refresh = 1;
}
break;
case SDL_QUIT:
case FF_QUIT_EVENT:
do_exit(cur_stream);
break;
default:
break;
}
}
do_exit(m_CurStream);
//m_CurStream = nullptr;
}