使用QOpenGLWidget显示
QT中的QOpenGLWidget底层是使用opengl库，是对opengl的再封装。使用QOpenGLWidget类时，我们仅需继承QOpenGLWidget，仅需重写void initializeGL()、void resizeGL(int w, int h)和void paintGL()三个函数即可。
目的与理论方法
我们的目的是为了在GPU端进行格式转换处理，然后显示视频流(图片)，减轻CPU的占用率。
我们将上诉的目的进行分解可知有以下几个工作：
- 定时(可选)；
- 格式转换，I420->RGB、NV12->RGB等；
- 显示图片(视频流)。
方法实现
将目的进行分解之后，我们对功能也就更明确了。接下来我们细说上述的三个待做工作：
定时功能实现
QT提供了多种定时器，我们选择一种自己常用的即可。例如如下代码：
#include <QTimer>
QTimer *t = new QTimer(this);
t->setInterval(50);
connect(t, &QTimer::timeout, [=](){
qDebug() << "timeout!";
};
t->start();
格式转换
能在显示器上显示的图像都为rgb图像。所以对于一些非rgb的图像格式，我们需要对其做格式转换，转为rgb格式。
对于一幅列(宽)为width、行(高)为height的图像数据来说：
rgba8888格式: 总大小为width * height * 4(bytes)。图像格式为rgbargbargba…rgba；
rgb888格式: 总大小为width * height * 3(bytes)。图像格式为rgbrgbrgb…rgb；
bgr888格式(OpenCV Mat格式): 总大小为width * height * 3(bytes)。图像格式为bgrbgr…bgr；
I420格式: 总大小为width * height * 1.5(bytes)。图像格式为y…yu…uv…v;
NV12格式:总大小为width * height * 1.5(bytes)。图像格式为y…yuvuvuv…uvuv;
NV21格式: 总大小为width * height * 1.5(bytes)。图像格式为y…yvuvu…vuvu;
Grayscale8格式: 总大小为width * height(bytes)。rgb三通道的值都一样。
…
了解了上述了几种常见的格式通道排布格式。我们就可以在cpu端进行运算操作然后将其转为rgb/rgba格式，然后进行显示；在QOpenGLWidget中，我们使用纹理映射来做相应的格式转换操作。
QOpenGLFunctions *f = QOpenGLContext::currentContext()->functions();
f->glActiveTexture(GL_TEXTURE0);
f->glBindTexture(GL_TEXTURE_2D, this->m->texture_id_y);
f->glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, this->m->m_video_w, this->m->m_video_h, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, this->m->m_video_ptr);
f->glActiveTexture(GL_TEXTURE1);
f->glBindTexture(GL_TEXTURE_2D, this->m->texture_id_u);
f->glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, this->m->m_video_w/2, this->m->m_video_h/2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, this->m->m_video_ptr+this->m->m_video_w*this->m->m_video_h);
f->glActiveTexture(GL_TEXTURE2);
f->glBindTexture(GL_TEXTURE_2D, this->m->texture_id_v);
f->glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, this->m->m_video_w/2, this->m->m_video_h/2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, this->m->m_video_ptr+this->m->m_video_w*this->m->m_video_h*5/4);
上述代码是I420格式的纹理读取操作，是在CPU端进行的，读取之后将数据送至GPU端。
" yuv.x = texture(texture_y, out_texture).r;\n"
" yuv.y = texture(texture_u, out_texture).r-0.5;\n"
" yuv.z = texture(texture_v, out_texture).r-0.5;\n"
" rgb = mat3(1, 1, 1, \
0, -0.34414, 1.77216,\
1.40168, -0.71417, 0) * yuv;\n"
" fragColor = vec4(rgb, alpha);\n"
上述代码是在片段着色器中进行运算的，该部分的操作是使用GPU运算，固可以极大的减少CPU的使用率。
显示视频流
在void initializeGL()函数中使用vao、vbo创建一个四边形及纹理坐标，然后再void paintGL()函数中更绑定纹理、更新纹理、绘制四边形进行纹理贴图即可。
完整代码
完整代码如下：
#ifndef VIDEOOPENGL_H
#define VIDEOOPENGL_H
#include <QOpenGLWidget>
#include <QOpenGLFunctions>
QT_BEGIN_NAMESPACE
class QOpenGLTexture;
class QOpenGLShaderProgram;
class QOpenGLBuffer;
class QOpenGLVertexArrayObject;
QT_END_NAMESPACE
namespace common {
namespace qt {
class VideoOpenGL : public QOpenGLWidget, protected QOpenGLFunctions
{
Q_OBJECT
public:
enum ImageFormat{
IMAGE_FORMAT_Unknown,
IMAGE_FORMAT_NV12, // y... uvuv...
IMAGE_FORMAT_NV21, // y... vuvu...
IMAGE_FORMAT_I420, // y... u... v...
IMAGE_FORMAT_RGB32, // b g r a
IMAGE_FORMAT_ARGB32, // b g r a
IMAGE_FORMAT_RGB888, // r g b
IMAGE_FORMAT_Grayscale8,// grayscale8
IMAGE_FORMAT_BGR888, // b g r(OpenCV Mat Format)
};
VideoOpenGL(QWidget* parent = nullptr);
~VideoOpenGL();
void set_image_size(const int& width, const int& height, const ImageFormat& farmat);
void update_image(char* data_ptr);
protected:
void initializeGL() override;
void paintGL() override;
void resizeGL(int w, int h) override;
void paintEvent(QPaintEvent *e) override;
private:
struct impl;
struct impl* m;
};
}
}
#endif // VIDEOOPENGL_H
#include "videoopengl.h"
#include <QOpenGLTexture>
#include <QOpenGLBuffer>
#include <QOpenGLVertexArrayObject>
#include <QOpenGLShaderProgram>
#include <iostream>
#include <QPainter>
using namespace common::qt;
QByteArray version_shader_code(const char* src);
struct VideoOpenGL::impl
{
impl() = default;
~impl(){
if(this->m_video_ptr != nullptr){
delete this->m_video_ptr;
this->m_video_ptr = nullptr;
}
f->glDeleteTextures(1, &texture_id_y);
f->glDeleteTextures(1, &texture_id_u);
f->glDeleteTextures(1, &texture_id_v);
m_vbo->release();
m_program->release();
delete m_vbo;
delete m_program;
}
QOpenGLShaderProgram *m_program ;
QOpenGLBuffer *m_vbo = nullptr;
QOpenGLVertexArrayObject *m_vao = nullptr;
GLuint texture_id_y, texture_id_u, texture_id_v;
int m_video_w = 0, m_video_h = 0;
int m_image_size = 0;
char* m_video_ptr = nullptr;
ImageFormat format = IMAGE_FORMAT_Unknown;
QOpenGLFunctions *f;
};
VideoOpenGL::VideoOpenGL(QWidget* parent)
: QOpenGLWidget(parent)
{
m = new struct impl;
}
VideoOpenGL::~VideoOpenGL()
{
makeCurrent();
delete m;
}
QByteArray version_shader_code(const char *src)
{
QByteArray versionedSrc;
if (QOpenGLContext::currentContext()->isOpenGLES())
versionedSrc.append(QByteArrayLiteral("#version 300 es\n"));
else
versionedSrc.append(QByteArrayLiteral("#version 330\n"));
versionedSrc.append(src);
return versionedSrc;
}
void VideoOpenGL::initializeGL()
{
//显示视频流，一定不要开启深度测试！！！
this->m->f = QOpenGLContext::currentContext()->functions();
QOpenGLFunctions *f = QOpenGLContext::currentContext()->functions();
const char* vertex_shader_source = "layout(location = 0) in vec4 in_position;\n"
"layout(location = 1) in vec2 in_texture;\n"
"out vec2 out_texture;\n"
"void main(void){\n"
" gl_Position = in_position;\n"
" out_texture = in_texture;\n"
"}\n";
// OpenGLES 内置矩阵mat3是一列一列的构建
const char* fragment_shader_source = "in highp vec2 out_texture;\n"
"uniform sampler2D texture_y;\n"
"uniform sampler2D texture_u;\n"
"uniform sampler2D texture_v;\n"
"uniform int video_format;\n"
"out highp vec4 fragColor;\n"
"void main(){\n"
" highp vec3 yuv, rgb;\n"
" highp float alpha = 1.0;\n"
" if(video_format == 1){\n"
" yuv.x = texture(texture_y, out_texture).r;\n"
" yuv.y = texture(texture_u, out_texture).r-0.5;\n"
" yuv.z = texture(texture_u, out_texture).a-0.5;\n"
" rgb = mat3(1, 1, 1, \
0, -0.34414, 1.77216,\
1.40168, -0.71417, 0) * yuv;\n"
" }else if(video_format == 2){\n"
" yuv.x = texture(texture_y, out_texture).r;\n"
" yuv.y = texture(texture_u, out_texture).a-0.5;\n"
" yuv.z = texture(texture_u, out_texture).r-0.5;\n"
" rgb = mat3(1, 1, 1, \
0, -0.34414, 1.77216,\
1.40168, -0.71417, 0) * yuv;\n"
" }else if(video_format == 3){\n"
" yuv.x = texture(texture_y, out_texture).r;\n"
" yuv.y = texture(texture_u, out_texture).r-0.5;\n"
" yuv.z = texture(texture_v, out_texture).r-0.5;\n"
" rgb = mat3(1, 1, 1, \
0, -0.34414, 1.77216,\
1.40168, -0.71417, 0) * yuv;\n"
" }else if(video_format == 4){\n"
" highp vec4 tmp = texture(texture_y, out_texture);\n"
" rgb = vec3(tmp.b, tmp.g, tmp.r);\n"
" }else if(video_format == 5){\n"
" highp vec4 tmp = texture(texture_y, out_texture);\n"
" rgb = vec3(tmp.b, tmp.g, tmp.r);\n"
" alpha = tmp.a;\n"
" }else if(video_format == 6){\n"
" highp vec4 tmp = texture(texture_y, out_texture);\n"
" rgb = vec3(tmp.r, tmp.g, tmp.b);\n"
" }else if(video_format == 7){\n"
" highp vec4 tmp = texture(texture_y, out_texture);\n"
" rgb = tmp.rgb;\n"
" }else if(video_format == 8){\n"
" highp vec4 tmp = texture(texture_y, out_texture);\n"
" rgb = vec3(tmp.b, tmp.g, tmp.r);\n"
" }\n"
" fragColor = vec4(rgb, alpha);\n"
"}\n";
this->m->m_program = new QOpenGLShaderProgram;
this->m->m_program->addShaderFromSourceCode(QOpenGLShader::Vertex, version_shader_code(vertex_shader_source));
this->m->m_program->addShaderFromSourceCode(QOpenGLShader::Fragment, version_shader_code(fragment_shader_source));
this->m->m_program->link();
this->m->m_program->bind();
this->m->m_program->setUniformValue("texture_y", 0);
this->m->m_program->setUniformValue("texture_u", 1);
this->m->m_program->setUniformValue("texture_v", 2);
#if 0
static const GLfloat vertex_vertices[] = {
-1.0f, -1.0f,
1.0f, -1.0f,
-1.0f, 1.0f,
1.0f, 1.0f,
};
static const GLfloat texture_vertices[] = {
0.0f, 1.0f,
1.0f, 1.0f,
0.0f, 0.0f,
1.0f, 0.0f,
};
f->glVertexAttribPointer(0, 2, GL_FLOAT, 0, 0, vertex_vertices);
f->glVertexAttribPointer(1, 2, GL_FLOAT, 0, 0, texture_vertices);
f->glEnableVertexAttribArray(0);
f->glEnableVertexAttribArray(1);
#else
static const GLfloat vertices[] = {
-1.0f, -1.0f, 0.0f, 1.0f,
1.0f, -1.0f, 1.0f, 1.0f,
-1.0f, 1.0f, 0.0f, 0.0f,
1.0f, 1.0f, 1.0f, 0.0f,
};
this->m->m_vao = new QOpenGLVertexArrayObject(this);
this->m->m_vao->create();
this->m->m_vao->bind();
this->m->m_vbo = new QOpenGLBuffer;
this->m->m_vbo->create();
this->m->m_vbo->bind();
this->m->m_vbo->allocate(vertices, sizeof(vertices));
f->glEnableVertexAttribArray(0);
f->glEnableVertexAttribArray(1);
f->glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 4*sizeof(float), 0);
f->glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 4*sizeof(float),
reinterpret_cast<void *>(2*sizeof(float)));
#endif
#if 1
f->glGenTextures(1, &this->m->texture_id_y);
f->glGenTextures(1, &this->m->texture_id_u);
f->glGenTextures(1, &this->m->texture_id_v);
f->glActiveTexture(GL_TEXTURE0);
f->glBindTexture(GL_TEXTURE_2D, this->m->texture_id_y);
f->glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_LINEAR);
f->glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR);
f->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
f->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
f->glActiveTexture(GL_TEXTURE1);
f->glBindTexture(GL_TEXTURE_2D, this->m->texture_id_u);
f->glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_LINEAR);
f->glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR);
f->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
f->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
f->glActiveTexture(GL_TEXTURE2);
f->glBindTexture(GL_TEXTURE_2D, this->m->texture_id_v);
f->glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_LINEAR);
f->glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR);
f->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
f->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
#endif
f->glBindTexture(GL_TEXTURE_2D, 0);
this->m->m_vao->release();
this->m->m_vbo->release();
this->m->m_program->release();
f->glEnable(GL_BLEND);
f->glBlendFunc(GL_SRC_ALPHA, GL_SRC_ALPHA_SATURATE);//GL_DST_ALPHA
}
void VideoOpenGL::resizeGL(int w, int h)
{
QOpenGLFunctions *f = QOpenGLContext::currentContext()->functions();
f->glViewport(0,0,w,h==0?1:h);
}
void VideoOpenGL::paintGL()
{
QOpenGLFunctions *f = QOpenGLContext::currentContext()->functions();
f->glClear(GL_COLOR_BUFFER_BIT);
this->m->m_program->bind();
this->m->m_vao->bind();
#if 1
this->m->m_program->setUniformValue("video_format", this->m->format);
switch (this->m->format)
{
case IMAGE_FORMAT_Unknown:
std::cout << "[W] Unknow video farmat." << std::endl;
return ;
case IMAGE_FORMAT_NV12:
case IMAGE_FORMAT_NV21:
f->glActiveTexture(GL_TEXTURE0);
f->glBindTexture(GL_TEXTURE_2D, this->m->texture_id_y);
f->glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, this->m->m_video_w, this->m->m_video_h, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, this->m->m_video_ptr);
f->glActiveTexture(GL_TEXTURE1);
f->glBindTexture(GL_TEXTURE_2D, this->m->texture_id_u);
f->glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE_ALPHA, this->m->m_video_w/2, this->m->m_video_h/2, 0, GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, this->m->m_video_ptr+this->m->m_video_w*this->m->m_video_h);
break;
case IMAGE_FORMAT_I420:
f->glActiveTexture(GL_TEXTURE0);
f->glBindTexture(GL_TEXTURE_2D, this->m->texture_id_y);
f->glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, this->m->m_video_w, this->m->m_video_h, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, this->m->m_video_ptr);
f->glActiveTexture(GL_TEXTURE1);
f->glBindTexture(GL_TEXTURE_2D, this->m->texture_id_u);
f->glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, this->m->m_video_w/2, this->m->m_video_h/2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, this->m->m_video_ptr+this->m->m_video_w*this->m->m_video_h);
f->glActiveTexture(GL_TEXTURE2);
f->glBindTexture(GL_TEXTURE_2D, this->m->texture_id_v);
f->glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, this->m->m_video_w/2, this->m->m_video_h/2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, this->m->m_video_ptr+this->m->m_video_w*this->m->m_video_h*5/4);
break;
case IMAGE_FORMAT_ARGB32:
case IMAGE_FORMAT_RGB32:
f->glActiveTexture(GL_TEXTURE0);
f->glBindTexture(GL_TEXTURE_2D, this->m->texture_id_y);
f->glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, this->m->m_video_w, this->m->m_video_h, 0, GL_RGBA, GL_UNSIGNED_BYTE, this->m->m_video_ptr);
break;
case IMAGE_FORMAT_RGB888:
f->glActiveTexture(GL_TEXTURE0);
f->glBindTexture(GL_TEXTURE_2D, this->m->texture_id_y);
f->glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, this->m->m_video_w, this->m->m_video_h, 0, GL_RGB, GL_UNSIGNED_BYTE, this->m->m_video_ptr);
break;
case IMAGE_FORMAT_Grayscale8:
f->glActiveTexture(GL_TEXTURE0);
f->glBindTexture(GL_TEXTURE_2D, this->m->texture_id_y);
f->glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, this->m->m_video_w, this->m->m_video_h, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, this->m->m_video_ptr);
break;
case IMAGE_FORMAT_BGR888:
f->glActiveTexture(GL_TEXTURE0);
f->glBindTexture(GL_TEXTURE_2D, this->m->texture_id_y);
f->glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, this->m->m_video_w, this->m->m_video_h, 0, GL_RGB, GL_UNSIGNED_BYTE, this->m->m_video_ptr);
break;
default:
break;
}
#endif
f->glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
f->glBindTexture(GL_TEXTURE_2D, 0);
this->m->m_vao->release();
this->m->m_program->release();
f->glFinish();
}
void VideoOpenGL::paintEvent(QPaintEvent *e)
{
QOpenGLWidget::paintEvent(e);
QPainter painter(this);
painter.fillRect(0, 0, 100, 100, Qt::green);
}
void VideoOpenGL::set_image_size(const int &width, const int &height, const ImageFormat& format)
{
this->m->m_video_w = width;
this->m->m_video_h = height;
this->m->format = format;
float multiple = 1.0;
switch (format)
{
case IMAGE_FORMAT_NV12:
case IMAGE_FORMAT_NV21:
case IMAGE_FORMAT_I420:
multiple = 1.5;
break;
case IMAGE_FORMAT_ARGB32:
case IMAGE_FORMAT_RGB32:
multiple = 4;
break;
case IMAGE_FORMAT_RGB888:
case IMAGE_FORMAT_BGR888:
multiple = 3;
break;
case IMAGE_FORMAT_Grayscale8:
multiple = 1;
break;
default:
break;
}
this->m->m_image_size = multiple*this->m->m_video_w*this->m->m_video_h;
if(this->m->m_video_ptr != nullptr){
delete this->m->m_video_ptr;
this->m->m_video_ptr = nullptr;
}
this->m->m_video_ptr = new char[this->m->m_image_size];
}
void VideoOpenGL::update_image(char *data_ptr)
{
memcpy(this->m->m_video_ptr, data_ptr, this->m->m_image_size);
update();
}
700
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
