【qml】qt 采用opengl渲染nv12视频到QSG并采用qml做显示

前面采用了离屏渲染，多线程的方式实现了nv12视频的播放，这次采用在QSG中渲染的方式实现，也就是在渲染线程中。

原理同样是将图像纹理到一个FBO中，然后通过qt的接口带到QSG中进行显示。

采用opengl渲染并用qml显示的接口类为QQuickFramebufferObject，渲染线程会在刷新画面的时候调用其createRenderer()方法用于创建渲染，这里我留了一个设置视频地址的接口，因为是显示视频嘛。

#ifndef VideoRende_H
#define VideoRende_H
#include

QT_FORWARD_DECLARE_CLASS(QTimer)
QT_FORWARD_DECLARE_CLASS(DecodeThread)
class VideoRender : public QQuickFramebufferObject
{
Q_OBJECT
Q_PROPERTY(QString videoSource READ videoSource WRITE setVideoSource NOTIFY videoSourceChanged) //增加一个设置视频地址的属性
public:
explicit VideoRender(QQuickItem* parent = nullptr);
~VideoRender();
Renderer* createRenderer() const override; //重写
void getFrame(uchar **nv12Ptr,int *w, int *h); //用于在创建的Renderer中访问当前帧的图像数据

signals:
void videoSourceChanged();

private:
QTimer *m_timer{nullptr};
QString m_videoSource;

QString videoSource();
void setVideoSource(QString);
DecodeThread *m_decodeThr{nullptr};

};

#endif // VideoRende_H
基实现如下

#include “VideoRender.h”
#include “nv12render.h”
#include “NvDecode/decodethread.h”
#include “logorenderer.h”
#include
#include
#include

class VideoFboRender : public QQuickFramebufferObject::Renderer //自定义一个FBO的渲染
{
public:
VideoFboRender(){
}

~VideoFboRender(){
}

//该函数主要从VideoRender中获取数据，用于在render()函数中的渲染;会先调用此函数再调用render
void synchronize(QQuickFramebufferObject*item){
    dynamic_cast<VideoRender*>(item)->getFrame(&nv12Ptr,&videoW,&videoH);
                       //此处的item是创建Nv12FboRender的QQuickFramebufferObject对象，在这里可以从QQuickFramebufferObject中获取数据保留下来，用于render中的渲染

// item->update(); //GUI线程中刷新,GUI阻塞，会影响画面,应该在QQuickFramebufferObject中用闹钟的方式刷新，避免阻塞，
//阻塞GUI线程和渲染线程都不好
}

void render()override{
    m_nv12Render.render(nv12Ptr,videoW,videoH); //渲染当前帧

// update(); //不是在GUI线程中刷新，GUI线程中的阻塞，不会影响画面;
}

QOpenGLFramebufferObject *createFramebufferObject(const QSize &size){
    QOpenGLFramebufferObjectFormat format;   //当大小发生变化时，会调用此函数生成对应大小的FBO
    format.setAttachment(QOpenGLFramebufferObject::CombinedDepthStencil);
    format.setSamples(4);
    return new QOpenGLFramebufferObject(size,format);
}

private:
Nv12Render m_nv12Render;
uchar *nv12Ptr{nullptr};
int videoW,videoH;
};

VideoRender::VideoRender(QQuickItem *parent):
QQuickFramebufferObject(parent)
{
m_timer = new QTimer(this);
m_timer->setInterval(30); //每过30ms刷新一次
connect(m_timer,SIGNAL(timeout()),this,SLOT(update()));

m_decodeThr = new DecodeThread;

}

VideoRender::~VideoRender()
{
m_decodeThr->requestInterruption();
m_decodeThr->quit();
m_decodeThr->wait();
m_decodeThr->deleteLater();
}

QQuickFramebufferObject::Renderer *VideoRender::createRenderer() const
{
return new VideoFboRender; //返回我们自定义的Renderer。
}

void VideoRender::getFrame(uchar **nv12Ptr, int *w, int *h) //返回当前帧数据
{
*nv12Ptr = m_decodeThr->imgPtr;
*w = m_decodeThr->videoW;
*h = m_decodeThr->videoH;
}

QString VideoRender::videoSource()
{
return m_videoSource;
}

void VideoRender::setVideoSource(QString url)
{
if(m_videoSource != url){
m_decodeThr->setUrl(url);
m_decodeThr->setTimeInterval(m_timer->interval());
m_decodeThr->start();
m_timer->start();
emit videoSourceChanged();
}

m_videoSource = url;

}
主要的结构就是这样了，接下来要说的是nv12的渲染了，这里和前面的nv12渲染有些不同，有几个坑。代码如下

#ifndef NV12RENDER_H
#define NV12RENDER_H
#include
#include

#include <QtGui/qvector3d.h>
#include <QtGui/qmatrix4x4.h>
#include <QtGui/qopenglshaderprogram.h>
#include <QtGui/qopenglfunctions.h>
#include

#include
#include

class Nv12Render : public QOpenGLFunctions
{
public:
Nv12Render();
void render(uchar*nv12Ptr, int w, int h); //渲染当前帧

private:
QOpenGLShaderProgram program;
GLuint idY,idUV; //Y分量和UV分量的纹理id
QOpenGLBuffer vbo; //用于在opengl服务端创建数据
qreal m_fAngle;
qreal m_fScale;
};

#endif // NV12RENDER_H
以下是其实现

#include “nv12render.h”
#include
#include
#include
#include <math.h>

Nv12Render::Nv12Render()
{
initializeOpenGLFunctions();
const char *vsrc =
“attribute vec4 vertexIn;
attribute vec4 textureIn;
varying vec4 textureOut;
uniform mediump mat4 matrix;
void main(void)
{
gl_Position = vertexIn * matrix;
textureOut = textureIn;
}”;

const char *fsrc =
        "varying mediump vec4 textureOut;\n"
        "uniform sampler2D textureY;\n"
        "uniform sampler2D textureUV;\n"
        "void main(void)\n"
        "{\n"
        "vec3 yuv; \n"
        "vec3 rgb; \n"
        "yuv.x = texture2D(textureY, textureOut.st).r - 0.0625; \n"
        "yuv.y = texture2D(textureUV, textureOut.st).r - 0.5; \n"
        "yuv.z = texture2D(textureUV, textureOut.st).g - 0.5; \n"
        "rgb = mat3( 1,       1,         1, \n"
                    "0,       -0.39465,  2.03211, \n"
                    "1.13983, -0.58060,  0) * yuv; \n"
        "gl_FragColor = vec4(rgb, 1); \n"
        "}\n";

program.addCacheableShaderFromSourceCode(QOpenGLShader::Vertex,vsrc);
program.addCacheableShaderFromSourceCode(QOpenGLShader::Fragment,fsrc);
program.link();

GLfloat points[]{
    -0.8f, 0.8f,
     0.8f, 0.8f,
     0.8f, -0.8f,
    -0.8f, -0.8f,

    0.0f,1.0f,
    1.0f,1.0f,
    1.0f,0.0f,
    0.0f,0.0f,
};

vbo.create();
vbo.bind();
vbo.allocate(points,sizeof(points));
//不能在这里直接将program绑定，以及vbo等等,感觉这里的program整个渲染的一部分,还有其它的program，必须在render()函数中进行数据的绑定和纹理的生成
GLuint ids[2];
glGenTextures(2,ids);
idY = ids[0];
idUV = ids[1];
m_fAngle = 0.0;
m_fScale = 1.0;

}

void Nv12Render::render(uchar *nv12Ptr, int w, int h)
{
glClearColor(0.5f, 0.5f, 0.7f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glDisable(GL_DEPTH_TEST);

program.bind(); //使用opengl切换为当前的program以使用shader
QMatrix4x4 modelview;
modelview.rotate(m_fAngle, 0.0f, 1.0f, 0.0f);
modelview.rotate(m_fAngle, 1.0f, 0.0f, 0.0f);
modelview.rotate(m_fAngle, 0.0f, 0.0f, 1.0f);
modelview.scale(m_fScale);
modelview.translate(0.0f, -0.2f, 0.0f);
program.setUniformValue("matrix", modelview);

vbo.bind(); //切换并使用创建好的顶点和纹理坐标数据
program.enableAttributeArray("vertexIn");
program.enableAttributeArray("textureIn");
program.setAttributeBuffer("vertexIn",GL_FLOAT, 0, 2, 2*sizeof(GLfloat));
program.setAttributeBuffer("textureIn",GL_FLOAT,2 * 4 * sizeof(GLfloat),2,2*sizeof(GLfloat));

glActiveTexture(GL_TEXTURE0 + 1); //这里我试了有个奇葩的现象，必须先选择1号纹理用来渲染Y分量数据，然后必须使用0号纹理来渲染UV分量的数据，原因我暂时也还没搞清楚
glBindTexture(GL_TEXTURE_2D,idY); //这真是一个大坑，我整了很久
glTexImage2D(GL_TEXTURE_2D,0,GL_RED,w,h,0,GL_RED,GL_UNSIGNED_BYTE,nv12Ptr);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);

glActiveTexture(GL_TEXTURE0 + 0); //使用0号纹理填充UV分量的数据
glBindTexture(GL_TEXTURE_2D,idUV);
glTexImage2D(GL_TEXTURE_2D,0,GL_RG,w >> 1,h >> 1,0,GL_RG,GL_UNSIGNED_BYTE,nv12Ptr + w*h);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);

program.setUniformValue("textureUV",0);
program.setUniformValue("textureY",1);
glDrawArrays(GL_QUADS,0,4);
program.disableAttributeArray("vertexIn");
program.disableAttributeArray("textureIn");
program.release(); //去掉program的绑定

m_fAngle += 1.0f;

}
以下为qml中的调用

import QtQuick 2.7
import QtQuick.Controls 2.0
import QtQuick.Layouts 1.3
import QtMultimedia 5.9
import SceneGraphRendering 1.0

ApplicationWindow {
visible: true
width: 640
height: 480
title: qsTr(“Hello World”)

Item{
    id: root
    anchors.fill: parent
    RenderItem{
        anchors.fill: parent
        videoSource: "D:/迅雷下载/香肠派对720p.mp4"
        Text {
            id: name2
            anchors.centerIn: parent
            font.pixelSize: 18
            color: "green"
            text: qsTr("texfffffffffffffffsft")
        }
    }

    MouseArea{
        anchors.fill: parent
        onClicked: {
            parent.grabToImage(function(result){
                console.log(result.image)

// result.saveToFile(“C:\Users\Administrator\Desktop\1.jpg”)
})
ani.start()
}
}

    Rectangle{
        id: rect
        width: parent.width * 0.7
        height: parent.height * 0.7
        anchors.centerIn: parent
        opacity: 0.5
        SequentialAnimation{
            id: ani
            ParallelAnimation{
                NumberAnimation{
                    target: rect
                    properties: "width"
                    to: rect.width ? 0 : root.width * 0.7
                    duration: 800
                }
                NumberAnimation{
                    target: rect
                    properties: "height"
                    to: rect.height ? 0 : root.height * 0.7
                    duration: 800
                }
            }
            ParallelAnimation{
                NumberAnimation{
                    target: rect
                    properties: "width"
                    to: rect.width ? 0 : root.width * 0.7
                    duration: 800
                }
                NumberAnimation{
                    target: rect
                    properties: "height"
                    to: rect.height ? 0 : root.height * 0.7
                    duration: 800
                }
            }
        }
    }
}

}
我自己定义了一个动画用于检测当抓图是画面是否会卡，实时证明，抓图是不会卡的，但是如果把抓出来的图片储到文件里面就会卡。