1、加入多线程
用主线程同时处理音视频编解码会导致视频出现卡顿,音频出现电音的情况。为避免这种情况,在项目中添加多线程,并将音频和视频解码分开能有效解决该问题。新建AudioInfo类来实现音频
解码和播放的逻辑,并将其放入一个独立的线程中。
为了配合多线程播放音视频的功能,你需要对VideoInfo
类进行以下调整:
-
确保
VideoInfo
类可以独立处理视频解码和渲染。 -
将视频解码和渲染放入独立的线程中运行,以避免与音频解码的线程发生冲突。
需要让VideoInfo
类继承自QThread
,以便它可以在单独的线程中运行。除此之外,代码结构保持不变,只是解码和渲染的逻辑将放入run()
方法中。
代码如下:
AudioInfo.h
#ifndef AUDIOINFO_H
#define AUDIOINFO_H
#include <QObject>
#include <QAudioSink>
#include <QBuffer>
#include<QThread>
extern "C" {
#include <libavformat/avformat.h>
#include <libavcodec/avcodec.h>
#include <libswresample/swresample.h>
#include <libavutil/opt.h>
#include <libavutil/channel_layout.h>
#include <libavutil/samplefmt.h>
}
class AudioInfo : public QObject
{
Q_OBJECT
public:
explicit AudioInfo(QObject *parent = nullptr);
~AudioInfo();
bool openFile(const QString &filePath);
bool initializeAudio();
void startPlayback();
void stopPlayback();
private:
AVFormatContext *formatContext;
AVCodecContext *codecContext;
AVFrame *frame;
AVPacket *packet;
SwrContext *swrContext;
int audioStreamIndex;
QAudioSink *audioSink;
QBuffer audioBuffer;
void decodeAndPlayAudio();
signals:
void playbackFinished();
};
#endif // AUDIOINFO_H
AudioInfo.cpp
#include "audioinfo.h"
#include <QDebug>
#include <QMediaDevices>
#include <QAudioSink>
#include <QBuffer>
#include <QThread>
AudioInfo::AudioInfo(QObject *parent)
: QObject(parent), formatContext(nullptr), codecContext(nullptr),
frame(nullptr), packet(nullptr), swrContext(nullptr),
audioStreamIndex(-1), audioSink(nullptr)
{
avformat_network_init();
}
AudioInfo::~AudioInfo()
{
if (frame) av_frame_free(&frame);
if (packet) av_packet_free(&packet);
if (codecContext) avcodec_free_context(&codecContext);
if (formatContext) avformat_close_input(&formatContext);
if (swrContext) swr_free(&swrContext);
if (audioSink) delete audioSink;
}
bool AudioInfo::openFile(const QString &filePath)
{
if (avformat_open_input(&formatContext, filePath.toStdString().c_str(), nullptr, nullptr) != 0) {
qWarning() << "Could not open file:" << filePath;
return false;
}
if (avformat_find_stream_info(formatContext, nullptr) < 0) {
qWarning() << "Could not find stream information";
return false;
}
av_dump_format(formatContext, 0, filePath.toStdString().c_str(), 0);
audioStreamIndex = -1;
for (unsigned int i = 0; i < formatContext->nb_streams; i++) {
if (formatContext->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
audioStreamIndex = i;
break;
}
}
if (audioStreamIndex == -1) {
qWarning() << "Could not find audio stream";
return false;
}
AVCodecParameters *codecParameters = formatContext->streams[audioStreamIndex]->codecpar;
const AVCodec *codec = avcodec_find_decoder(codecParameters->codec_id);
if (!codec) {
qWarning() << "Unsupported codec!";
return false;
}
codecContext = avcodec_alloc_context3(codec);
if (!codecContext) {
qWarning() << "Could not allocate codec context";
return false;
}
if (avcodec_parameters_to_context(codecContext, codecParameters) < 0) {
qWarning() << "Could not copy codec parameters to codec context";
return false;
}
if (avcodec_open2(codecContext, codec, nullptr) < 0) {
qWarning() << "Could not open codec";
return false;
}
frame = av_frame_alloc();
packet = av_packet_alloc();
if (!initializeAudio()) { // 调用 initializeAudio
qWarning() << "Could not initialize audio";
return false;
}
return true;
}
bool AudioInfo::initializeAudio()
{
QAudioFormat format;
format.setSampleRate(codecContext->sample_rate);
format.setChannelCount(codecContext->ch_layout.nb_channels);
// 确保使用正确的音频格式
switch (codecContext->sample_fmt) {
case AV_SAMPLE_FMT_FLT:
case AV_SAMPLE_FMT_FLTP:
format.setSampleFormat(QAudioFormat::Float);
break;
case AV_SAMPLE_FMT_S16:
case AV_SAMPLE_FMT_S16P:
format.setSampleFormat(QAudioFormat::Int16);
break;
default:
qWarning() << "Unsupported sample format!";
return false;
}
if (!QMediaDevices::audioOutputs().isEmpty()) {
QAudioDevice audioOutputDevice = QMediaDevices::defaultAudioOutput();
audioSink = new QAudioSink(audioOutputDevice, format, this);
if (!audioBuffer.isOpen()) {
audioBuffer.open(QIODevice::ReadWrite);
}
audioSink->start(&audioBuffer);
qDebug() << "Audio initialized with Sample Rate:" << format.sampleRate()
<< "Channels:" << format.channelCount()
<< "Sample Format:" << format.sampleFormat();
return true;
} else {
qWarning() << "No available audio output devices!";
return false;
}
}
void AudioInfo::startPlayback()
{
qDebug() << "startPlayback is running";
while (av_read_frame(formatContext, packet) >= 0) {
if (packet->stream_index == audioStreamIndex) {
int response = avcodec_send_packet(codecContext, packet);
if (response >= 0) {
response = avcodec_receive_frame(codecContext, frame);
if (response >= 0) {
decodeAndPlayAudio();
}
}
av_packet_unref(packet);
// 如果 audioSink 处于非活跃状态,等待其变为活跃状态
if (audioSink->bytesFree() < 8192) {
if (audioSink->state() == QAudio::IdleState || audioSink->state() == QAudio::StoppedState) {
qDebug() << "Waiting for audio sink to become active...";
while (audioSink->state() != QAudio::ActiveState) {
QThread::msleep(10);
}
}
}
}
}
emit playbackFinished();
}
void AudioInfo::stopPlayback()
{
if (audioSink) {
audioSink->stop();
audioBuffer.close();
}
}
void AudioInfo::decodeAndPlayAudio()
{
int data_size = av_samples_get_buffer_size(nullptr, codecContext->ch_layout.nb_channels,
frame->nb_samples, codecContext->sample_fmt, 1);
audioBuffer.write((const char*)frame->data[0], data_size);
qDebug() << "Writing audio data to buffer, size:" << data_size
<< "buffer position:" << audioBuffer.pos();
}
VideoInfo.h
// videoinfo.h
#ifndef VIDEOINFO_H
#define VIDEOINFO_H
#include <QObject>
#include <QImage>
#include <QTimer>
#include <QBuffer>
#include <QAudioSink> // 新增
#include <QMediaDevices> // 新增
#include <QThread>
#include <QMutex>
extern "C" {
#include <libavformat/avformat.h>
#include <libavcodec/avcodec.h>
#include <libswscale/swscale.h>
#include <libavutil/imgutils.h>
#include <libswresample/swresample.h>
#include <libavutil/channel_layout.h>
#include <libavutil/opt.h>
}
class VideoInfo : public QThread
{
Q_OBJECT
public:
explicit VideoInfo(QObject *parent = nullptr);
~VideoInfo();
bool openFile(const QString &filePath);
//QImage getFrame();
void startPlayback();
void stopPlayback();
// void processAudioData();
signals:
void frameReady(const QImage &image);
protected:
void run();
private:
AVFormatContext *formatContext;
AVCodecContext *videoCodecContext;
AVFrame *videoFrame;
AVPacket *videoPacket;
SwsContext *swsContext;
int videoStreamIndex;
bool playing;
QMutex mutex; // 用于线程安全的锁
};
#endif // VIDEOINFO_H
VideoInfo.cpp
#include "videoinfo.h"
#include <QDebug>
#include <QImage>
VideoInfo::VideoInfo(QObject *parent)
: QThread(parent), formatContext(nullptr), videoCodecContext(nullptr),
videoFrame(nullptr), videoPacket(nullptr), swsContext(nullptr),
videoStreamIndex(-1), playing(false)
{
}
VideoInfo::~VideoInfo()
{
if (videoFrame) av_frame_free(&videoFrame);
if (videoPacket) av_packet_free(&videoPacket);
if (videoCodecContext) avcodec_free_context(&videoCodecContext);
if (formatContext) avformat_close_input(&formatContext);
if (swsContext) sws_freeContext(swsContext);
}
bool VideoInfo::openFile(const QString &filePath)
{
// 打开文件并找到视频流
if (avformat_open_input(&formatContext, filePath.toStdString().c_str(), nullptr, nullptr) != 0) {
qWarning() << "Could not open file:" << filePath;
return false;
}
if (avformat_find_stream_info(formatContext, nullptr) < 0) {
qWarning() << "Could not find stream information";
return false;
}
videoStreamIndex = -1;
for (unsigned int i = 0; i < formatContext->nb_streams; i++) {
if (formatContext->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
videoStreamIndex = i;
break;
}
}
if (videoStreamIndex == -1) {
qWarning() << "Could not find video stream";
return false;
}
AVCodecParameters *codecParameters = formatContext->streams[videoStreamIndex]->codecpar;
const AVCodec *videoCodec = avcodec_find_decoder(codecParameters->codec_id);
if (!videoCodec) {
qWarning() << "Unsupported codec!";
return false;
}
videoCodecContext = avcodec_alloc_context3(videoCodec);
if (!videoCodecContext) {
qWarning() << "Could not allocate codec context";
return false;
}
if (avcodec_parameters_to_context(videoCodecContext, codecParameters) < 0) {
qWarning() << "Could not copy codec parameters to codec context";
return false;
}
if (avcodec_open2(videoCodecContext, videoCodec, nullptr) < 0) {
qWarning() << "Could not open codec";
return false;
}
// 初始化视频转换器
swsContext = sws_getContext(videoCodecContext->width, videoCodecContext->height, videoCodecContext->pix_fmt,
videoCodecContext->width, videoCodecContext->height, AV_PIX_FMT_RGB24,
SWS_BILINEAR, nullptr, nullptr, nullptr);
if (!swsContext) {
qWarning() << "Could not initialize SwsContext";
return false;
}
return true;
}
void VideoInfo::startPlayback()
{
mutex.lock();
playing = true;
mutex.unlock();
start(); // 启动线程
}
void VideoInfo::stopPlayback()
{
mutex.lock();
playing = false;
mutex.unlock();
}
void VideoInfo::run()
{
videoFrame = av_frame_alloc();
videoPacket = av_packet_alloc();
while (playing) {
if (av_read_frame(formatContext, videoPacket) >= 0) {
if (videoPacket->stream_index == videoStreamIndex) {
int response = avcodec_send_packet(videoCodecContext, videoPacket);
if (response >= 0) {
response = avcodec_receive_frame(videoCodecContext, videoFrame);
if (response >= 0) {
// 转换视频帧格式
QImage image(videoCodecContext->width, videoCodecContext->height, QImage::Format_RGB888);
uint8_t *dest[4] = { image.bits(), nullptr, nullptr, nullptr };
int destLinesize[4] = { static_cast<int>(image.bytesPerLine()), 0, 0, 0 };
sws_scale(swsContext, videoFrame->data, videoFrame->linesize, 0, videoCodecContext->height,
dest, destLinesize);
emit frameReady(image);
}
}
}
av_packet_unref(videoPacket);
} else {
break;
}
}
av_frame_free(&videoFrame);
av_packet_free(&videoPacket);
}
运行结果:
总结
-
多线程处理:
VideoInfo
和AudioInfo
类各自运行在独立的线程中,实现音视频解码和播放的并行处理。 -
线程控制:
startPlayback()
和stopPlayback()
用于控制视频的播放和停止,run()
方法内执行视频解码和渲染的主循环。 -
线程安全:通过
QMutex
来确保线程之间的同步与安全操作。
2、OpenGL 显示视频
通过 OpenGL 显示视频并取代 QLabel,需要将解码后的视频帧作为纹理渲染到屏幕上。
使用 OpenGL 方法渲染视频帧的步骤:
-
创建纹理:将视频帧数据上传到 OpenGL 纹理中。
-
设置着色器:使用简单的着色器程序来绘制纹理。
-
绘制纹理:使用顶点缓冲区和纹理坐标将纹理绘制到屏幕上。
步骤如下:
1. 初始化 OpenGL 环境
需要创建一个继承自 QOpenGLWidget
的类来处理 OpenGL 渲染,创建并初始化一个 QOpenGLShaderProgram
,从文件加载顶点和片段着色器,并链接它们,通过着色器程序来进行视频帧的渲染处理;创建纹理对象 texture
,并设置其最小化、放大过滤器,以及纹理的边缘处理模式。代码如下:
void VideoRenderer::initializeGL() {
initializeOpenGLFunctions();
glClearColor(0.0, 0.0, 0.0, 1.0); // 设置背景色
// 创建着色器程序
shaderProgram = new QOpenGLShaderProgram();
shaderProgram->addShaderFromSourceFile(QOpenGLShader::Vertex, ":/vertex.vert");
shaderProgram->addShaderFromSourceFile(QOpenGLShader::Fragment, ":/fragment.frag");
shaderProgram->link();
// 设置纹理
texture = new QOpenGLTexture(QOpenGLTexture::Target2D);
texture->setMinificationFilter(QOpenGLTexture::Linear);
texture->setMagnificationFilter(QOpenGLTexture::Linear);
texture->setWrapMode(QOpenGLTexture::ClampToEdge);
}
片段着色器和顶点着色器代码如下:
#version 330 core
out vec4 FragColor;
in vec2 TexCoord;
uniform sampler2D texture1;
void main()
{
FragColor = texture(texture1, TexCoord);
}
#version 330 core
layout (location = 0) in vec3 aPos;
layout (location = 1) in vec2 aTexCoord;
out vec2 TexCoord;
void main()
{
gl_Position = vec4(aPos, 1.0);
TexCoord = aTexCoord;
}
2. 绘制视频帧
创建 OpenGL 纹理,设置纹理参数,将视频帧数据上传到纹理,启用纹理和绘制四边形
3.调整视频大小并保持比例
通过计算合适的比例因子,基于 VideoRenderer
的高度来调整视频的大小,使得视频的高度和 VideoRenderer
的高度一致,并且保持宽高比不变;获取 VideoRenderer
的高度和宽度。
获取视频帧的高度和宽度。计算等比例缩放系数,使视频帧的高度适配 VideoRenderer
的高度。根据缩放系数调整视频帧的宽度,确保其宽高比不变。使用 glViewport
来设定适当的渲染区域。
绘制视频帧和调整视频大小并保持比例的代码如下:
void VideoRenderer::paintGL()
{
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
if (!currentFrame.isNull()) {
// 获取 VideoRenderer 和 视频帧的大小
int rendererWidth = width();
int rendererHeight = height();
int videoWidth = currentFrame.width();
int videoHeight = currentFrame.height();
// 计算缩放比例,使视频的高度和 VideoRenderer 的高度一致
float scaleFactor = static_cast<float>(rendererHeight) / videoHeight;
// 计算缩放后的视频宽度
int scaledVideoWidth = static_cast<int>(videoWidth * scaleFactor);
int scaledVideoHeight = rendererHeight; // 缩放后的高度等于 VideoRenderer 的高度
// 计算居中显示所需的偏移量(水平居中)
int xOffset = (rendererWidth - scaledVideoWidth) / 2;
// 创建 OpenGL 纹理
GLuint textureID;
glGenTextures(1, &textureID);
glBindTexture(GL_TEXTURE_2D, textureID);
// 设置纹理参数
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
// 将视频帧数据上传到纹理
QImage glImage = currentFrame.convertToFormat(QImage::Format_RGBA8888);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, glImage.width(), glImage.height(), 0, GL_RGBA, GL_UNSIGNED_BYTE, glImage.bits());
// 启用纹理和绘制四边形
glEnable(GL_TEXTURE_2D);
glBegin(GL_QUADS);
// 左下角
glTexCoord2f(0.0f, 1.0f);
glVertex2f(-1.0f + 2.0f * xOffset / rendererWidth, -1.0f);
// 右下角
glTexCoord2f(1.0f, 1.0f);
glVertex2f(-1.0f + 2.0f * (xOffset + scaledVideoWidth) / rendererWidth, -1.0f);
// 右上角
glTexCoord2f(1.0f, 0.0f);
glVertex2f(-1.0f + 2.0f * (xOffset + scaledVideoWidth) / rendererWidth, 1.0f);
// 左上角
glTexCoord2f(0.0f, 0.0f);
glVertex2f(-1.0f + 2.0f * xOffset / rendererWidth, 1.0f);
glEnd();
glDisable(GL_TEXTURE_2D);
// 删除纹理
glDeleteTextures(1, &textureID);
}
}
运行结果如下:
总结
- 顶点着色器负责将顶点坐标和纹理坐标传递到OpenGL管线。
- 片段着色器使用传入的纹理坐标从纹理中采样颜色并渲染到屏幕上。
- 您需要将每一帧视频作为纹理上传,并在
paintGL()
中使用 OpenGL 管线进行绘制。 - 通过缩放视频的高度来适配
VideoRenderer
的高度。如果是相反的情况,可以按宽度来适配,而不是高度。