文章目录
说在前面
- opencv版本：4.0.1
- opencv aruco版本：4.0.1
- opengl：使用glad、glfw
- ar实现：基于标记(marker)
- visual studio版本：2017
- 原理部分：【OpenCV&OpenGL&AR】原理部分
实验结果
说明
- OpenGL部分不熟悉的话建议看看LearnOpenGL教程，只需要看第一章 [入门] 以及这一小节的内容即可。
OpenGL部分的代码多是LearnOpenGL教程的源码 - OpenCV aruco官方有部分教程，我也写了部分栗子。
- 建议与原理部分一起食用。
代码
-
头文件
//glad&glfw #include <glad/glad.h> #include <GLFW/glfw3.h> //GLM是OpenGL Mathematics的缩写，它是一个只有头文件的库， //也就是说我们只需包含对应的头文件就行了，不用链接和编译 //具体请参考LearnOpenGL教程 #include <glm\glm.hpp> #include <glm\gtc\matrix_transform.hpp> #include <glm\gtc\type_ptr.hpp> //opencv部分，主要是aruco #include <opencv2\core.hpp> #include <opencv2\core\opengl.hpp> #include <opencv2\core\cuda.hpp> #include <opencv2\highgui.hpp> #include <opencv2\aruco.hpp> #include <opencv2\imgproc.hpp> #include <opencv2\calib3d.hpp> //stb_image主要用于读取纹理； //stb_image.h是Sean Barrett的一个非常流行的单头文件图像加载库， //它能够加载大部分流行的文件格式，并且能够很简单得整合到你的工程之中 //具体请参考LearnOpenGL教程 #include "stb_image.h" //shader.h定义了一个Shader类，封装了读取并编译着色器程序等过程 #include "shader.h" #include <iostream> -
初始化OpenGL
// // glfw: initialize and configure // ------------------------------ glfwInit(); glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 3); glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 3); glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE); // glfw window creation // -------------------- GLFWwindow* window = glfwCreateWindow(SCR_WIDTH, SCR_HEIGHT, "LearnOpenGL", NULL, NULL); if (window == NULL) { std::cout << "Failed to create GLFW window" << std::endl; glfwTerminate(); return -1; } glfwMakeContextCurrent(window); glfwSetFramebufferSizeCallback(window, framebuffer_size_callback); // glad: load all OpenGL function pointers // --------------------------------------- if (!gladLoadGLLoader((GLADloadproc)glfwGetProcAddress)) { std::cout << "Failed to initialize GLAD" << std::endl; return -1; }// glfw: whenever the window size changed //(by OS or user resize) this callback function executes // -------------------------------------------------------------------------------- void framebuffer_size_callback(GLFWwindow* window, int width, int height) { // make sure the viewport matches the new window dimensions; note that width and // height will be significantly larger than specified on retina displays. glViewport(0, 0, width, height); } -
正射投影的准备工作
- 首先准备一个矩形
- OpenCV捕获的图像会作为纹理贴在这个矩形上
- 最后投影到OpenGL相机
//准备矩形，后两步在后面 // build and compile our shader zprogram // ------------------------------------ Shader texShader("texture.vs", "texture.fs"); // set up vertex data (and buffer(s)) and configure vertex attributes // ------------------------------------------------------------------ float tex_vertices[] = { // positions // colors // texture coords 1.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 1.0f, 1.0f, // top right 1.0f, -1.0f, 0.0f, 0.0f, 1.0f, 0.0f, 1.0f, 0.0f, // bottom right -1.0f, -1.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, // bottom left -1.0f, 1.0f, 0.0f, 1.0f, 1.0f, 0.0f, 0.0f, 1.0f // top left }; unsigned int indices[] = { 0, 1, 3, // first triangle 1, 2, 3 // second triangle }; unsigned int TVBO, TVAO, TEBO; glGenVertexArrays(1, &TVAO); glGenBuffers(1, &TVBO); glGenBuffers(1, &TEBO); glBindVertexArray(TVAO); glBindBuffer(GL_ARRAY_BUFFER, TVBO); glBufferData(GL_ARRAY_BUFFER, sizeof(tex_vertices), tex_vertices, GL_STATIC_DRAW); glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, TEBO); glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(indices), indices, GL_STATIC_DRAW); // position attribute glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(float), (void*)0); glEnableVertexAttribArray(0); // color attribute glVertexAttribPointer(1, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(float), (void*)(3 * sizeof(float))); glEnableVertexAttribArray(1); // texture coord attribute glVertexAttribPointer(2, 2, GL_FLOAT, GL_FALSE, 8 * sizeof(float), (void*)(6 * sizeof(float))); glEnableVertexAttribArray(2); -
准备立方体
- 关于Uniform BLock
具体见高级GLSL
Shader ourShader("shader.vs", "shader.fs"); // set up vertex data (and buffer(s)) and configure vertex attributes // ------------------------------------------------------------------ float cube_w = 0.5f; float vertices[] = { -cube_w+cube_w, -cube_w+cube_w, -cube_w+cube_w, 0.0f, 0.0f, //... //这里省略了部分数据，具体数据见源码 -cube_w+cube_w, cube_w+cube_w, -cube_w+cube_w, 0.0f, 1.0f }; unsigned int VBO, VAO; glGenVertexArrays(1, &VAO); glGenBuffers(1, &VBO); glBindVertexArray(VAO); glBindBuffer(GL_ARRAY_BUFFER, VBO); glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW); // position attribute glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 5 * sizeof(float), (void *)0); glEnableVertexAttribArray(0); // texture coord attribute glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 5 * sizeof(float), (void *)(3 * sizeof(float))); glEnableVertexAttribArray(1); // // Uniform Block // unsigned int uniformBlockIndex = glGetUniformBlockIndex(ourShader.ID, "Matrices"); glUniformBlockBinding(ourShader.ID, uniformBlockIndex, 0); glGenBuffers(1, &matricesUniBuffer); glBindBuffer(GL_UNIFORM_BUFFER, matricesUniBuffer); glBufferData(GL_UNIFORM_BUFFER, MatricesUniBufferSize, NULL, GL_DYNAMIC_DRAW); glBindBufferRange(GL_UNIFORM_BUFFER, 0, matricesUniBuffer, 0, MatricesUniBufferSize); //setUniforms(); glBindBuffer(GL_UNIFORM_BUFFER, 0); - 关于Uniform BLock
-
准备立方体的纹理
// load and create a texture // ------------------------- unsigned int texture1; // texture 1 // --------- glGenTextures(1, &texture1); glBindTexture(GL_TEXTURE_2D, texture1); // set the texture wrapping parameters glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT); // set texture filtering parameters glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); // load image, create texture and generate mipmaps int width, height, nrChannels; stbi_set_flip_vertically_on_load(true); // tell stb_image.h to flip loaded texture's on the y-axis. unsigned char *data = stbi_load("container.jpg", &width, &height, &nrChannels, 0); if (data) { glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, width, height, 0, GL_RGB, GL_UNSIGNED_BYTE, data); glGenerateMipmap(GL_TEXTURE_2D); } else { std::cout << "Failed to load texture" << std::endl; } stbi_image_free(data); // tell opengl for each sampler to which texture unit it belongs to (only has to be done once) // ------------------------------------------------------------------------------------------- ourShader.use(); ourShader.setInt("texture1", 0); -
读取相机参数
void readCameraPara() { dictionary = cv::aruco::getPredefinedDictionary(aruco::PREDEFINED_DICTIONARY_NAME(0)); cv::FileStorage fs("camera.yml", cv::FileStorage::READ); fs["camera_matrix"] >> camera_matrix; fs["distortion_coefficients"] >> dist_coeffs; std::cout << "camera_matrix\n" << camera_matrix << std::endl; std::cout << "\ndist coeffs\n" << dist_coeffs << std::endl; } -
构造Projection Matrix
static Mat_<float> projMatrix; void buildProjectionMatrix(float nearp, float farp) { projMatrix.create(4, 4); projMatrix.setTo(0); float f_x = camera_matrix.at<double>(0, 0); float f_y = camera_matrix.at<double>(1, 1); float c_x = camera_matrix.at<double>(0, 2); float c_y = camera_matrix.at<double>(1, 2); projMatrix.at<float>(0, 0) = 2 * f_x / (float)SCR_WIDTH; projMatrix.at<float>(1, 1) = 2 * f_y / (float)SCR_HEIGHT; projMatrix.at<float>(2, 0) = 1.0f - 2 * c_x / (float)SCR_WIDTH; projMatrix.at<float>(2, 1) = 2 * c_y / (float)SCR_HEIGHT - 1.0f; projMatrix.at<float>(2, 2) = -(farp + nearp) / (farp - nearp); projMatrix.at<float>(2, 3) = -1.0f; projMatrix.at<float>(3, 2) = -2.0f*farp*nearp / (farp - nearp); glBindBuffer(GL_UNIFORM_BUFFER, matricesUniBuffer); glBufferSubData(GL_UNIFORM_BUFFER, ProjMatrixOffset, MatrixSize, projMatrix.data); glBindBuffer(GL_UNIFORM_BUFFER, 0); } -
检测每一帧中的标记并计算View Matrix
void detectArucoMarkers(cv::Mat &image) { cv::aruco::detectMarkers( image, //输入 dictionary, //标记字典 markerCorners, //标记顶点坐标 markerIds, //标记ID detectorParams, rejectedCandidates); if (markerIds.size() > 0) { cv::aruco::drawDetectedMarkers(image, markerCorners, markerIds); std::vector< cv::Vec3d > rvecs, tvecs; cv::aruco::estimatePoseSingleMarkers( markerCorners, // 标记的顶点坐标 markerLength, // 标记边长 camera_matrix, //相机内参 dist_coeffs, //畸变系数 rvecs, // 旋转向量 tvecs); // 平移向量 for (unsigned int i = 0; i < markerIds.size(); i++) { cv::Vec3d r = rvecs[i]; cv::Vec3d t = tvecs[i]; cv::Mat viewMatrixf = cv::Mat::zeros(4, 4, CV_32F); cv::Mat rot; //构造View Matrix Rodrigues(rvecs[i], rot); for (unsigned int row = 0; row < 3; ++row) { for (unsigned int col = 0; col < 3; ++col) { viewMatrixf.at<float>(row, col) = (float)rot.at<double>(row, col); } viewMatrixf.at<float>(row, 3) = (float)tvecs[i][row];// *0.1f; } viewMatrixf.at<float>(3, 3) = 1.0f; //反转Y、Z轴 cv::Mat cvToGl = cv::Mat::zeros(4, 4, CV_32F); cvToGl.at<float>(0, 0) = 1.0f; cvToGl.at<float>(1, 1) = -1.0f; // Invert the y axis cvToGl.at<float>(2, 2) = -1.0f; // invert the z axis cvToGl.at<float>(3, 3) = 1.0f; viewMatrixf = cvToGl * viewMatrixf; //以列為主元素，记得原理部分的注意事项吗？ cv::transpose(viewMatrixf, viewMatrixf); viewMatrix = viewMatrixf; //画坐标轴，OpenCV坐标系，可有可无 cv::aruco::drawAxis(image, camera_matrix, dist_coeffs, r, t, 0.5*markerLength); } is_mark = true; } else { is_mark = false; } } -
准备矩形的纹理并渲染
// load and create a texture // ------------------------- unsigned int texture; glGenTextures(1, &texture); glBindTexture(GL_TEXTURE_2D, texture); // set the texture wrapping parameters glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT); // set texture filtering parameters glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); // load image, create texture and generate mipmaps Mat frame; cap >> frame; detectArucoMarkers(frame); cv::flip(frame, frame, 0);//这里要反转一下，不然是镜像 glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, frame.cols, frame.rows, 0, GL_BGR, GL_UNSIGNED_BYTE, frame.data); glGenerateMipmap(GL_TEXTURE_2D); // render // ------ glClearColor(0.2f, 0.3f, 0.3f, 1.0f); glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); // bind Texture glBindTexture(GL_TEXTURE_2D, texture); // render container texShader.use(); glBindVertexArray(TVAO); glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0); -
渲染立方体
setModelMatrix();//设置M0del Matrix setCamera(viewMatrix);//设置View Matrix //Projection Matrix设置一次就行，放在循环外面 // bind textures on corresponding texture units glActiveTexture(GL_TEXTURE0); glBindTexture(GL_TEXTURE_2D, texture1); // activate shader ourShader.use(); // render box if (is_mark) { glBindVertexArray(VAO); glDrawArrays(GL_TRIANGLES, 0, 36); } -
其他
- 例如释放缓存啥的
遇到的一些问题
- 问题如下，立方体在侧面的时侯就像移动了一样！
出错原因： 这个立方体的顶点坐标是这个亚子的，也就是说该立方体有一半是在y轴下面，但是我们在投影的时候先是将opencv捕捉的图像正射投影到OpenGL相机中，然后再投影OpenGL世界中的立方体，所以立方体一定是在OpenCV捕捉的图像上面，也就是说整个立方体都会显示出来，这就导致了上面的结果。
因此，在设置OpenGL中的物体的时候，一定要注意将对象放在y正半轴。
float vertices[] = {
-0.5f, -0.5f, -0.5f, 0.0f, 0.0f,
0.5f, -0.5f, -0.5f, 1.0f, 0.0f,
0.5f, 0.5f, -0.5f, 1.0f, 1.0f,
0.5f, 0.5f, -0.5f, 1.0f, 1.0f,
-0.5f, 0.5f, -0.5f, 0.0f, 1.0f,
-0.5f, -0.5f, -0.5f, 0.0f, 0.0f,
-0.5f, -0.5f, 0.5f, 0.0f, 0.0f,
0.5f, -0.5f, 0.5f, 1.0f, 0.0f,
0.5f, 0.5f, 0.5f, 1.0f, 1.0f,
0.5f, 0.5f, 0.5f, 1.0f, 1.0f,
-0.5f, 0.5f, 0.5f, 0.0f, 1.0f,
-0.5f, -0.5f, 0.5f, 0.0f, 0.0f,
-0.5f, 0.5f, 0.5f, 1.0f, 0.0f,
-0.5f, 0.5f, -0.5f, 1.0f, 1.0f,
-0.5f, -0.5f, -0.5f, 0.0f, 1.0f,
-0.5f, -0.5f, -0.5f, 0.0f, 1.0f,
-0.5f, -0.5f, 0.5f, 0.0f, 0.0f,
-0.5f, 0.5f, 0.5f, 1.0f, 0.0f,
0.5f, 0.5f, 0.5f, 1.0f, 0.0f,
0.5f, 0.5f, -0.5f, 1.0f, 1.0f,
0.5f, -0.5f, -0.5f, 0.0f, 1.0f,
0.5f, -0.5f, -0.5f, 0.0f, 1.0f,
0.5f, -0.5f, 0.5f, 0.0f, 0.0f,
0.5f, 0.5f, 0.5f, 1.0f, 0.0f,
-0.5f, -0.5f, -0.5f, 0.0f, 1.0f,
0.5f, -0.5f, -0.5f, 1.0f, 1.0f,
0.5f, -0.5f, 0.5f, 1.0f, 0.0f,
0.5f, -0.5f, 0.5f, 1.0f, 0.0f,
-0.5f, -0.5f, 0.5f, 0.0f, 0.0f,
-0.5f, -0.5f, -0.5f, 0.0f, 1.0f,
-0.5f, 0.5f, -0.5f, 0.0f, 1.0f,
0.5f, 0.5f, -0.5f, 1.0f, 1.0f,
0.5f, 0.5f, 0.5f, 1.0f, 0.0f,
0.5f, 0.5f, 0.5f, 1.0f, 0.0f,
-0.5f, 0.5f, 0.5f, 0.0f, 0.0f,
-0.5f, 0.5f, -0.5f, 0.0f, 1.0f
};
源码
-
Github
求Star- 工程文件
这个工程里包含了opencv库以及opengl，相关配置已经调好了，但是由于opencv库编译用的vs2017，所以运行的话可能也需要2017，当然也可以自己更改opencv的配置，有问题请留言！ - (2020/09/09补充)工程配置
可执行目录：AR_Cube\AR_Cube\opencv\x86\vc15\bin
包含目录：AR_Cube\AR_Cube\opencv\include、AR_Cube\AR_Cube\opengl\include
库目录：AR_Cube\AR_Cube\opencv\x86\vc15\lib、AR_Cube\AR_Cube\opengl\lib、AR_Cube\AR_Cube\opencv\x86\vc15\bin
系统环境变量：AR_Cube\AR_Cube\opencv\x86\vc15\bin
若还有问题请在评论区提出
- 工程文件
-
CSDN(
暂时没放，看看能不能把工程文件放上去)
END
2万+
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
