Android基于RTMP视频流的人脸识别（下篇）

在上篇中我们已经通过ffmpeg将Rtmp流成功的解析为NV21数据，接下来笔者将讲述如何通过OpenGL将NV21绘制上去，以及如何通过NV21进行人脸识别，并绘制人脸框。

1、OpenGL的数据绘制

OpenGL的绘制需要设置好顶点着色器与片段着色器，笔者提供了一个着色器工具类GLUtil,里面定义了普通效果，灰度效果与浮雕效果，可以供大家参考。除此之外还要根据frame宽高设置好纹理数据。最后就是将裸数据传入后提取Y、U、V分量，再绘制上去即可。笔者自定义了一个RtmpGLSurfaceView给大家作一个参考。

public class RtmpGLSurfaceView extends GLSurfaceView {
    private static final String TAG = "CameraGLSurfaceView";
    // 源视频帧宽/高
    private int frameWidth, frameHeight;
    private boolean isMirror;
    private int rotateDegree = 0;
    // 用于判断preview数据是否被传入，避免在初始化时有一段时间的绿色背景（y、u、v均全为0）
    private boolean dataInput = false;
    // 圆角半径
    private int radius = 0;
​
    private ByteBuffer yBuf = null, uBuf = null, vBuf = null;
    // 纹理id
    private int[] yTexture = new int[1];
    private int[] uTexture = new int[1];
    private int[] vTexture = new int[1];
​
    //YUV分量
    private byte[] yArray;
    private byte[] uArray;
    private byte[] vArray;
​
    private static final int FLOAT_SIZE_BYTES = 4;
    //片段着色器的效果
    private String fragmentShaderCode = GLUtil.FRAG_SHADER_NORMAL;
​
    private FloatBuffer squareVertices = null;
    private FloatBuffer coordVertices = null;
    private boolean rendererReady = false;
    float[] coordVertice = null;
​
    public RtmpGLSurfaceView(Context context) {
        this(context, null);
    }
​
    public RtmpGLSurfaceView(Context context, AttributeSet attrs) {
        super(context, attrs);
        setEGLContextClientVersion(2);
        //设置Renderer到GLSurfaceView
        setRenderer(new YUVRenderer());
        // 只有在绘制数据改变时才绘制view
        setRenderMode(RENDERMODE_WHEN_DIRTY);
        setOutlineProvider(new ViewOutlineProvider() {
            @Override
            public void getOutline(View view, Outline outline) {
                Rect rect = new Rect(0, 0, view.getMeasuredWidth(), view.getMeasuredHeight());
                outline.setRoundRect(rect, radius);
            }
        });
        setClipToOutline(true);
    }
​
    public void turnRound() {
        invalidateOutline();
    }
​
    public int getRadius() {
        return radius;
    }
​
    public void setRadius(int radius) {
        this.radius = radius;
    }
​
    /**
     * 设置不同的片段着色器代码以达到不同的预览效果
     *
     * @param fragmentShaderCode 片段着色器代码
     */
    public void setFragmentShaderCode(String fragmentShaderCode) {
        this.fragmentShaderCode = fragmentShaderCode;
    }
​
    public void init(boolean isMirror, int rotateDegree, int frameWidth, int frameHeight) {
        if (this.frameWidth == frameWidth
                && this.frameHeight == frameHeight
                && this.rotateDegree == rotateDegree
                && this.isMirror == isMirror) {
            return;
        }
        dataInput = false;
        this.frameWidth = frameWidth;
        this.frameHeight = frameHeight;
        this.rotateDegree = rotateDegree;
        this.isMirror = isMirror;
        yArray = new byte[this.frameWidth * this.frameHeight];
        uArray = new byte[this.frameWidth * this.frameHeight / 4];
        vArray = new byte[this.frameWidth * this.frameHeight / 4];
​
        int yFrameSize = this.frameHeight * this.frameWidth;
        int uvFrameSize = yFrameSize >> 2;
        yBuf = ByteBuffer.allocateDirect(yFrameSize);
        yBuf.order(ByteOrder.nativeOrder()).position(0);
​
        uBuf = ByteBuffer.allocateDirect(uvFrameSize);
        uBuf.order(ByteOrder.nativeOrder()).position(0);
​
        vBuf = ByteBuffer.allocateDirect(uvFrameSize);
        vBuf.order(ByteOrder.nativeOrder()).position(0);
        // 顶点坐标
        squareVertices = ByteBuffer
                .allocateDirect(GLUtil.SQUARE_VERTICES.length * FLOAT_SIZE_BYTES)
                .order(ByteOrder.nativeOrder())
                .asFloatBuffer();
        squareVertices.put(GLUtil.SQUARE_VERTICES).position(0);
        //纹理坐标
        if (isMirror) {
            switch (rotateDegree) {
                case 0:
                    coordVertice = GLUtil.MIRROR_COORD_VERTICES;
                    break;
                case 90:
                    coordVertice = GLUtil.ROTATE_90_MIRROR_COORD_VERTICES;
                    break;
                case 180:
                    coordVertice = GLUtil.ROTATE_180_MIRROR_COORD_VERTICES;
                    break;
                case 270:
                    coordVertice = GLUtil.ROTATE_270_MIRROR_COORD_VERTICES;
                    break;
                default:
                    break;
            }
        } else {
            switch (rotateDegree) {
                case 0:
                    coordVertice = GLUtil.COORD_VERTICES;
                    break;
                case 90:
                    coordVertice = GLUtil.ROTATE_90_COORD_VERTICES;
                    break;
                case 180:
                    coordVertice = GLUtil.ROTATE_180_COORD_VERTICES;
                    break;
                case 270:
                    coordVertice = GLUtil.ROTATE_270_COORD_VERTICES;
                    break;
                default:
                    break;
            }
        }
        coordVertices = ByteBuffer.allocateDirect(coordVertice.length * FLOAT_SIZE_BYTES).order(ByteOrder.nativeOrder()).asFloatBuffer();
        coordVertices.put(coordVertice).position(0);
    }
​
    /**
     * 创建OpenGL Program并关联GLSL中的变量
     *
     * @param fragmentShaderCode 片段着色器代码
     */
    private void createGLProgram(String fragmentShaderCode) {
        int programHandleMain = GLUtil.createShaderProgram(fragmentShaderCode);
        if (programHandleMain != -1) {
            // 使用着色器程序
            GLES20.glUseProgram(programHandleMain);
            // 获取顶点着色器变量
            int glPosition = GLES20.glGetAttribLocation(programHandleMain, "attr_position");
            int textureCoord = GLES20.glGetAttribLocation(programHandleMain, "attr_tc");
​
            // 获取片段着色器变量
            int ySampler = GLES20.glGetUniformLocation(programHandleMain, "ySampler");
            int uSampler = GLES20.glGetUniformLocation(programHandleMain, "uSampler");
            int vSampler = GLES20.glGetUniformLocation(programHandleMain, "vSampler");
​
            //给变量赋值
            /**
             * GLES20.GL_TEXTURE0 和 ySampler 绑定
             * GLES20.GL_TEXTURE1 和 uSampler 绑定
             * GLES20.GL_TEXTURE2 和 vSampler 绑定
             *
             * 也就是说 glUniform1i的第二个参数代表图层序号
             */
            GLES20.glUniform1i(ySampler, 0);
            GLES20.glUniform1i(uSampler, 1);
            GLES20.glUniform1i(vSampler, 2);
​
            GLES20.glEnableVertexAttribArray(glPosition);
            GLES20.glEnableVertexAttribArray(textureCoord);
​
            /**
             * 设置Vertex Shader数据
             */
            squareVertices.position(0);
            GLES20.glVertexAttribPointer(glPosition, GLUtil.COUNT_PER_SQUARE_VERTICE, GLES20.GL_FLOAT, false, 8, squareVertices);
            coordVertices.position(0);
            GLES20.glVertexAttribPointer(textureCoord, GLUtil.COUNT_PER_COORD_VERTICES, GLES20.GL_FLOAT, false, 8, coordVertices);
        }
    }
​
    public class YUVRenderer implements Renderer {
        private void initRenderer() {
            rendererReady = false;
            createGLProgram(fragmentShaderCode);
​
            //启用纹理
            GLES20.glEnable(GLES20.GL_TEXTURE_2D);
            //创建纹理
            createTexture(frameWidth, frameHeight, GLES20.GL_LUMINANCE, yTexture);
            createTexture(frameWidth / 2, frameHeight / 2, GLES20.GL_LUMINANCE, uTextur