Android OpenGLES3绘图 - 使用CameraX

最新推荐文章于 2024-04-23 17:59:21 发布

yihanss

最新推荐文章于 2024-04-23 17:59:21 发布

阅读量877

点赞数

分类专栏： Android 文章标签： android kotlin android studio 音视频

本文链接：https://blog.csdn.net/yihanss/article/details/128105511

版权

Android 专栏收录该内容

62 篇文章 1 订阅

订阅专栏

相机处理是OpenGL一个重要的应用场景，因为OpenGL的主要工作是处理图像，而相机每秒生成几十帧图像，用GPU来处理再合适不过了。

至于Android CameraX和OpenGL的结合使用，网上有不少教程了，然而它们都有一个特点，就是给两者增加了不必要的耦合。由于两者本身架构都设计得非常好，实际上它们只需要一点耦合：就是OpenGL给Camera提供一个Surface。

如果分别实现了CameraX和OpenGL的Texture纹理图片，那么只需要改动极少的代码就能把两者结合起来。

在这里插入图片描述

1 CameraX使用

详见教程https://www.jianshu.com/p/f79855586ee2
在一个MainActivity中就能写完全部代码


typealias LumaListener = (luma: Double) -> Unit

class MainActivity : AppCompatActivity() {
    private var imageCapture: ImageCapture? = null

    private lateinit var outputDirectory: File
    private lateinit var cameraExecutor: ExecutorService

    override fun onCreate(savedInstanceState: Bundle?) {
        super.onCreate(savedInstanceState)
        setContentView(R.layout.activity_main)

        if (allPermissionsGranted()) {
            startCamera()
        } else {
            ActivityCompat.requestPermissions(this, REQUIRED_PERMISSIONS, REQUEST_CODE_PERMISSIONS)
        }

        camera_capture_button.setOnClickListener { takePhoto() }

        outputDirectory = getOutputDirectory()

        cameraExecutor = Executors.newSingleThreadExecutor()
    }

    private class LuminosityAnalyzer(private val listener : LumaListener) : ImageAnalysis.Analyzer {

        private fun ByteBuffer.toByteArray() : ByteArray {
            rewind()
            val data = ByteArray(remaining())
            get(data)
            return data
        }

        override fun analyze(image: ImageProxy) {
            val buffer = image.planes[0].buffer
            val data = buffer.toByteArray()
            val pixels = data.map { it.toInt() and 0xFF }
            val luma = pixels.average()

            listener(luma)

            image.close()
        }

    }

    private fun takePhoto() {
        val imageCapture = imageCapture ?: return

        val photoFile = File(
                outputDirectory, SimpleDateFormat(FILENAME_FORMAT, Locale.US).format(System.currentTimeMillis()) + ".jpg"
        )

        val outputOptions = ImageCapture.OutputFileOptions.Builder(photoFile).build()

        imageCapture.takePicture(
                outputOptions, ContextCompat.getMainExecutor(this), object : ImageCapture.OnImageSavedCallback {
            override fun onImageSaved(outputFileResults: ImageCapture.OutputFileResults) {
                val savedUri = Uri.fromFile(photoFile)
                val msg = "Photo captured: $savedUri"
                Toast.makeText(baseContext, msg, Toast.LENGTH_SHORT).show()
                Log.d("chao", msg)
            }

            override fun onError(exception: ImageCaptureException) {
                exception.printStackTrace()
            }
        })
    }

    private fun startCamera() {
        val cameraProviderFuture = ProcessCameraProvider.getInstance(this)

        cameraProviderFuture.addListener(Runnable {
            // 1\. 选择摄像头，相机预览
            val cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA
            val cameraProvider: ProcessCameraProvider = cameraProviderFuture.get()

            val preview = Preview.Builder()
                .setTargetResolution(Size(480, 640))
                .build().also {
                it.setSurfaceProvider(viewFinder.createSurfaceProvider())
            }

            // 2\. 相机拍照
            imageCapture = ImageCapture.Builder().build()

            // 3\. 数据处理
            val imageAnalyzer = ImageAnalysis.Builder().build().also {
                it.setAnalyzer(cameraExecutor, LuminosityAnalyzer {luma ->
                    Log.d("chao", "Average luminosity: $luma")
                })
            }

            try {
                cameraProvider.unbindAll()

                cameraProvider.bindToLifecycle(this, cameraSelector, preview, imageCapture, imageAnalyzer)

            } catch (e: Exception) {
                e.printStackTrace()
            }

        }, ContextCompat.getMainExecutor(this))

    }

    private fun allPermissionsGranted() = REQUIRED_PERMISSIONS.all {
        ContextCompat.checkSelfPermission(baseContext, it) == PackageManager.PERMISSION_GRANTED
    }

    private fun getOutputDirectory(): File {
        val mediaDir = externalMediaDirs.firstOrNull()?.let {
            File(it, resources.getString(R.string.app_name)).apply { mkdirs() }
        }
        return if (mediaDir != null && mediaDir.exists()) mediaDir else filesDir
    }

    override fun onDestroy() {
        cameraExecutor.shutdown()
        super.onDestroy()
    }

    override fun onRequestPermissionsResult(requestCode: Int, permissions: Array<out String>, grantResults: IntArray) {
        if (requestCode == REQUEST_CODE_PERMISSIONS) {
            if (allPermissionsGranted()) {
                startCamera()
            } else {
                Toast.makeText(this,"Permissions not granted by the user.",
                        Toast.LENGTH_SHORT).show()
                finish()
            }
        }
    }

    companion object {
        private const val TAG = "CameraXBasic"
        private const val FILENAME_FORMAT = "yyyy-MM-dd-HH-mm-ss-SSS"
        private const val REQUEST_CODE_PERMISSIONS = 10
        private val REQUIRED_PERMISSIONS = arrayOf(Manifest.permission.CAMERA)
    }

2 OpenGL绘制Texture图片

顶点数据里添加坐标、颜色和纹理坐标；
创建VAO、VBO和EBO；
创建GL_TEXTURE_2D纹理，用glUniform1i传给Shader；
绘制时用glUniformMatrix4fv传入变化矩阵给Shader，因为不需要变化，所以暂时用单元矩阵。

package com.example.opengles3camera

import android.graphics.Bitmap
import android.graphics.SurfaceTexture
import android.opengl.*
import android.opengl.GLES20.*
import androidx.camera.core.Preview
import androidx.camera.core.SurfaceRequest
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.nio.FloatBuffer
import java.nio.IntBuffer
import javax.microedition.khronos.egl.EGLConfig
import javax.microedition.khronos.opengles.GL10

class CameraRender: GLSurfaceView.Renderer {

    var vertices = floatArrayOf( //     ---- 位置 ----       ---- 颜色 ----     - 纹理坐标 -
        0.5f, 0.5f, 0.0f, 1.0f, 0.0f, 0.0f, 1.0f, 1.0f,  // 右上
        0.5f, -0.5f, 0.0f, 0.0f, 1.0f, 0.0f, 1.0f, 0.0f,  // 右下
        -0.5f, -0.5f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f,  // 左下
        -0.5f, 0.5f, 0.0f, 1.0f, 1.0f, 0.0f, 0.0f, 1.0f // 左上
    )

    val indices = intArrayOf( // 注意索引从0开始!
        0, 1, 3,  // 第一个三角形
        1, 2, 3 // 第二个三角形
    )

    var program = 0
    var vertexBuffer: FloatBuffer? = null
    var intBuffer: IntBuffer? = null
    var vao: IntArray = IntArray(1)
    var tex: IntArray = IntArray(1)

    override fun onSurfaceCreated(gl: GL10?, config: EGLConfig?) {
        program = ShaderUtils.loadProgram()
        //分配内存空间,每个浮点型占4字节空间
        vertexBuffer = ByteBuffer.allocateDirect(vertices.size * 4)
            .order(ByteOrder.nativeOrder())
            .asFloatBuffer()
        //传入指定的坐标数据
        vertexBuffer!!.put(vertices)
        vertexBuffer!!.position(0)
        vao = IntArray(1)
        GLES30.glGenVertexArrays(1, vao, 0)
        GLES30.glBindVertexArray(vao[0])
        val vbo = IntArray(1)
        glGenBuffers(1, vbo, 0)
        glBindBuffer(GL_ARRAY_BUFFER, vbo[0])
        glBufferData(GL_ARRAY_BUFFER, vertices.size * 4, vertexBuffer, GL_STATIC_DRAW)

        intBuffer = IntBuffer.allocate(indices.size * 4)
        intBuffer!!.put(indices)
        intBuffer!!.position(0)
        val ebo = IntArray(1)
        glGenBuffers(1, ebo, 0)
        glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo[0])
        glBufferData(GL_ELEMENT_ARRAY_BUFFER, indices.size * 4, intBuffer, GL_STATIC_DRAW)

        glGenTextures(1, tex, 0)
        glActiveTexture(GL_TEXTURE0)
        glBindTexture(GL_TEXTURE_2D, tex[0])
        // 为当前绑定的纹理对象设置环绕、过滤方式
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT)
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT)
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR)
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)
        val bitmap: Bitmap = ShaderUtils.loadImageAssets("face.png")
        GLUtils.texImage2D(GL_TEXTURE_2D, 0, bitmap, 0)
        glGenerateMipmap(GL_TEXTURE_2D)

        glUseProgram(program)
        val loc0 = glGetUniformLocation(program, "texture1")
        glUniform1i(loc0, 0)

        // Load the vertex data
        glVertexAttribPointer(0, 3, GL_FLOAT, false, 8 * 4, 0)
        glEnableVertexAttribArray(0)
        glVertexAttribPointer(1, 3, GL_FLOAT, false, 8 * 4, 3 * 4)
        glEnableVertexAttribArray(1)
        glVertexAttribPointer(2, 2, GL_FLOAT, false, 8 * 4, 6 * 4)
        glEnableVertexAttribArray(2)
        glBindBuffer(GL_ARRAY_BUFFER, 0)
        GLES30.glBindVertexArray(0)
        glClearColor(0.5f, 0.5f, 0.5f, 0.5f)
    }

    override fun onSurfaceChanged(gl: GL10?, width: Int, height: Int) {
        glViewport(0, 0, width, height)
    }

    var transform = FloatArray(16)

    override fun onDrawFrame(gl: GL10?) {
        // Clear the color buffer
        glClear(GL_COLOR_BUFFER_BIT)

        // Use the program object
        glUseProgram(program)
        glBindTexture(GL_TEXTURE_2D, tex[0])

        Matrix.setIdentityM(transform, 0)
        //        Matrix.translateM(transform, 0, 0, 0, 0);

        val loc = glGetUniformLocation(program, "transform")
        glUniformMatrix4fv(loc, 1, false, transform, 0)
        GLES30.glBindVertexArray(vao[0])

//            glDrawArrays ( GL_TRIANGLES, 0, vertices.length );
        glDrawElements(GL_TRIANGLES, vertices.size, GL_UNSIGNED_INT, 0)
    }

}

在MainActivity中做一点修改，创建GLSurfaceView添加到Activity中。

    override fun onCreate(savedInstanceState: Bundle?) {
        super.onCreate(savedInstanceState)

        glSurfaceView = GLSurfaceView(this)
        val screenW = resources.displayMetrics.widthPixels
        glSurfaceView.layoutParams = FrameLayout.LayoutParams(screenW, screenW * 4 / 3)
        setContentView(glSurfaceView)

        glSurfaceView.setEGLContextClientVersion(3)
        cameraRender = CameraRender()
        glSurfaceView.setRenderer(cameraRender)
        glSurfaceView.renderMode = GLSurfaceView.RENDERMODE_CONTINUOUSLY

        if (allPermissionsGranted()) {
            startCamera()
        } else {
            ActivityCompat.requestPermissions(this, REQUIRED_PERMISSIONS, REQUEST_CODE_PERMISSIONS)
        }

//        camera_capture_button.setOnClickListener { takePhoto() }

        outputDirectory = getOutputDirectory()

        cameraExecutor = Executors.newSingleThreadExecutor()
    }

3 用OpenGL预览相机

修改相机预览目标
将startCamera方法中setSurfaceProvider参数改成cameraRender

    private fun startCamera() {
        val cameraProviderFuture = ProcessCameraProvider.getInstance(this)

        cameraProviderFuture.addListener(Runnable {
            // 1\. 选择摄像头，相机预览
            val cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA
            val cameraProvider: ProcessCameraProvider = cameraProviderFuture.get()

            val preview = Preview.Builder()
                .setTargetResolution(Size(480, 640))
                .build().also {
//                it.setSurfaceProvider(viewFinder.createSurfaceProvider())
                it.setSurfaceProvider(cameraRender)
            }

cameraRender创建OESTexture，实现Preview.SurfaceProvider接口
将原来创建GL_TEXTURE_2D的方法改成创建GL_TEXTURE_EXTERNAL_OES，并创建一个SurfaceTexture对象；


    var surfaceTexture: SurfaceTexture? = null

    private val executor = Executors.newSingleThreadExecutor()

    override fun onSurfaceCreated(gl: GL10?, config: EGLConfig?) {
    ...
//        glGenTextures(1, tex, 0)
//        glActiveTexture(GL_TEXTURE0)
//        glBindTexture(GL_TEXTURE_2D, tex[0])
//        // 为当前绑定的纹理对象设置环绕、过滤方式
//        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT)
//        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT)
//        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR)
//        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)
//        val bitmap: Bitmap = ShaderUtils.loadImageAssets("face.png")
//        GLUtils.texImage2D(GL_TEXTURE_2D, 0, bitmap, 0)
//        glGenerateMipmap(GL_TEXTURE_2D)

        tex = createOESTexture()
        surfaceTexture = SurfaceTexture(tex[0])
        surfaceTexture?.setOnFrameAvailableListener {
//            requestRender()
        }
}

    fun createOESTexture(): IntArray {
        val arr = IntArray(1)
        glGenTextures(1, arr, 0)
        glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, arr[0])
        glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR.toFloat())
        glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR.toFloat())
        glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE.toFloat())
        glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE.toFloat())
        return arr
    }

实现Preview.SurfaceProvider接口的onSurfaceRequested方法，根据方法提供的参数设置surfaceTexture的BufferSize，然后用surfaceTexture创建一个Surface，将它提供给SurfaceRequest。

class CameraRender: GLSurfaceView.Renderer, Preview.SurfaceProvider {
    ...
    override fun onSurfaceRequested(request: SurfaceRequest) {
        val size = request.resolution
        surfaceTexture?.setDefaultBufferSize(size.width, size.height)
        val surface = Surface(surfaceTexture)
        request.provideSurface(surface, executor, Consumer {
            surfaceTexture?.release()
            surface.release()
        })
    }
}

绘制
在绘制前调用surfaceTexture?.updateTexImage()更新画面，然后获取坐标变换矩阵给transform。

    override fun onDrawFrame(gl: GL10?) {
        // Clear the color buffer
        glClear(GL_COLOR_BUFFER_BIT)

        surfaceTexture?.updateTexImage()
        surfaceTexture?.getTransformMatrix(transform)

        // Use the program object
        glUseProgram(program)
        glBindTexture(GL_TEXTURE_2D, tex[0])

//        Matrix.setIdentityM(transform, 0)
//        Matrix.translateM(transform, 1, 1f, 0f, 0f);

        var loc = glGetUniformLocation(program, "transform")
        glUniformMatrix4fv(loc, 1, false, transform, 0)
        GLES30.glBindVertexArray(vao[0])

//            glDrawArrays ( GL_TRIANGLES, 0, vertices.length );
        glDrawElements(GL_TRIANGLES, vertices.size, GL_UNSIGNED_INT, 0)
    }

Shader
在顶点着色器中用transform对纹理坐标进行转换。

#version 300 es
layout (location = 0) in vec3 vPosition;
layout (location = 1) in vec3 vColor;
layout (location = 2) in vec2 vTexCoord;
out vec3 aColor;
out vec2 aTexCoord;

uniform mat4 transform;

void main() {
     gl_Position = vec4(vPosition, 1.0f);
     aColor = vColor;
     aTexCoord = (transform * vec4(vTexCoord, 1.0f, 1.0f)).xy;
}

在片段着色器中将sampler2D改成samplerExternalOES，添加必要的extension。

#version 300 es
#extension GL_OES_EGL_image_external : require
#extension GL_OES_EGL_image_external_essl3 : require

precision mediump float;
in vec3 aColor;
in vec2 aTexCoord;
out vec4 fragColor;

uniform samplerExternalOES texture1;

void main() {

     // 正常画面
     fragColor = texture(texture1, aTexCoord);
}

这样就能看到相机画面了。

4 实现简单滤镜

在片段着色器中稍作处理就能实现一些滤镜效果。

    // 彩色滤镜效果
    fragColor = mix(texture(texture1, aTexCoord), vec4(aColor, 1.0f), 0.5f);

在这里插入图片描述

     // 四分屏效果
     float s = aTexCoord.s;
     float t = aTexCoord.t;
     if (s > 0.5f) {
          s = s - 0.5f;
     }
     if (t > 0.5f) {
          t = t - 0.5f;
     }
     fragColor = texture(texture1, vec2(s, t));

在这里插入图片描述

     // 马赛克效果
     float s = aTexCoord.s;
     float t = aTexCoord.t;
     float msk = 50.0f;
     s = float(int(s * msk)) / msk;
     t = float(int(t * msk)) / msk;
     fragColor = texture(texture1, vec2(s, t));

在这里插入图片描述

5 Github地址

完整项目在SurfacePaint项目下的opengles3camera模块里。

原文作者：rome753
链接：https://www.jianshu.com/p/bcabdb0fc65a

yihanss

关注

0
点赞
踩
2

收藏

觉得还不错? 一键收藏
0
评论
Android OpenGLES3绘图 - 使用CameraX

相机处理是OpenGL一个重要的应用场景，因为OpenGL的主要工作是处理图像，而相机每秒生成几十帧图像，用GPU来处理再合适不过了。至于Android CameraX和OpenGL的结合使用，网上有不少教程了，然而它们都有一个特点，就是给两者增加了不必要的耦合。由于两者本身架构都设计得非常好，实际上它们只需要一点耦合：就是OpenGL给Camera提供一个Surface。如果分别实现了CameraX和OpenGL的Texture纹理图片，那么只需要改动极少的代码就能把两者结合起来。
复制链接

扫一扫