一、环境
pycharm+OpenGL+opengame
二、静态及动态增强现实实现
(一)静态(立方体和茶壶)
1.立方体的实现
将静态立方体放到键盘上
效果图:
(a)
(b)
(c)
图 一
代码:
![](https://i-blog.csdnimg.cn/blog_migrate/8f900a89c6347c561fdf2122f13be562.gif)
![](https://i-blog.csdnimg.cn/blog_migrate/961ddebeb323a10fe0623af514929fc1.gif)
1 from pylab import * 2 from PIL import Image 3 # If you have PCV installed, these imports should work 4 from PCV.geometry import homography, camera 5 from PCV.localdescriptors import sift 6 7 """ 8 This is the augmented reality and pose estimation cube example from Section 4.3. 9 """ 10 11 def draw_teapot(size): 12 glEnable(GL_LIGHTING) 13 glEnable(GL_LIGHT0) 14 def cube_points(c, wid): 15 """ Creates a list of points for plotting 16 a cube with plot. (the first 5 points are 17 the bottom square, some sides repeated). """ 18 p = [] 19 # bottom 20 p.append([c[0]-wid, c[1]-wid, c[2]-wid]) 21 p.append([c[0]-wid, c[1]+wid, c[2]-wid]) 22 p.append([c[0]+wid, c[1]+wid, c[2]-wid]) 23 p.append([c[0]+wid, c[1]-wid, c[2]-wid]) 24 p.append([c[0]-wid, c[1]-wid, c[2]-wid]) #same as first to close plot 25 26 # top 27 p.append([c[0]-wid, c[1]-wid, c[2]+wid]) 28 p.append([c[0]-wid, c[1]+wid, c[2]+wid]) 29 p.append([c[0]+wid, c[1]+wid, c[2]+wid]) 30 p.append([c[0]+wid, c[1]-wid, c[2]+wid]) 31 p.append([c[0]-wid, c[1]-wid, c[2]+wid]) #same as first to close plot 32 33 # vertical sides 34 p.append([c[0]-wid, c[1]-wid, c[2]+wid]) 35 p.append([c[0]-wid, c[1]+wid, c[2]+wid]) 36 p.append([c[0]-wid, c[1]+wid, c[2]-wid]) 37 p.append([c[0]+wid, c[1]+wid, c[2]-wid]) 38 p.append([c[0]+wid, c[1]+wid, c[2]+wid]) 39 p.append([c[0]+wid, c[1]-wid, c[2]+wid]) 40 p.append([c[0]+wid, c[1]-wid, c[2]-wid]) 41 42 return array(p).T 43 44 45 def my_calibration(sz): 46 """ 47 Calibration function for the camera (iPhone4) used in this example. 48 """ 49 row, col = sz 50 fx = 2555*col/2592 51 fy = 2586*row/1936 52 K = diag([fx, fy, 1]) 53 K[0, 2] = 0.5*col 54 K[1, 2] = 0.5*row 55 return K 56 57 58 59 # compute features 60 sift.process_image('1.png', 'im0.sift') 61 l0, d0 = sift.read_features_from_file('im0.sift') 62 63 sift.process_image('1.png', 'im1.sift') 64 l1, d1 = sift.read_features_from_file('im1.sift') 65 66 67 # match features and estimate homography 68 matches = sift.match_twosided(d0, d1) 69 ndx = matches.nonzero()[0] 70 fp = homography.make_homog(l0[ndx, :2].T) 71 ndx2 = [int(matches[i]) for i in ndx] 72 tp = homography.make_homog(l1[ndx2, :2].T) 73 74 model = homography.RansacModel() 75 H, inliers = homography.H_from_ransac(fp, tp, model) 76 77 # camera calibration 78 K = my_calibration((747, 1000)) 79 80 # 3D points at plane z=0 with sides of length 0.2 81 box = cube_points([0, 0, 0.1], 0.1) 82 83 # project bottom square in first image 84 cam1 = camera.Camera(hstack((K, dot(K, array([[0], [0], [-1]]))))) 85 # first points are the bottom square 86 box_cam1 = cam1.project(homography.make_homog(box[:, :5])) 87 88 89 # use H to transfer points to the second image 90 box_trans = homography.normalize(dot(H,box_cam1)) 91 92 # compute second camera matrix from cam1 and H 93 cam2 = camera.Camera(dot(H, cam1.P)) 94 A = dot(linalg.inv(K), cam2.P[:, :3]) 95 A = array([A[:, 0], A[:, 1], cross(A[:, 0], A[:, 1])]).T 96 cam2.P[:, :3] = dot(K, A) 97 98 # project with the second camera 99 box_cam2 = cam2.project(homography.make_homog(box)) 100 101 102 103 # plotting 104 im0 = array(Image.open('1.png')) 105 im1 = array(Image.open('2.png')) 106 107 figure() 108 imshow(im0) 109 plot(box_cam1[0, :], box_cam1[1, :], linewidth=3) 110 title('2D projection of bottom square') 111 axis('off') 112 113 figure() 114 imshow(im1) 115 plot(box_trans[0, :], box_trans[1, :], linewidth=3) 116 title('2D projection transfered with H') 117 axis('off') 118 119 figure() 120 imshow(im1) 121 plot(box_cam2[0, :], box_cam2[1, :], linewidth=3) 122 title('3D points projected in second image') 123 axis('off') 124 125 show()
2.茶壶的实现
将茶壶放到书籍上
效果图:
图二
代码:
![](https://i-blog.csdnimg.cn/blog_migrate/8f900a89c6347c561fdf2122f13be562.gif)
![](https://i-blog.csdnimg.cn/blog_migrate/961ddebeb323a10fe0623af514929fc1.gif)
import math import pickle from pylab import * from OpenGL.GL import * from OpenGL.GLU import * from OpenGL.GLUT import * import pygame, pygame.image from pygame.locals import * from PCV.geometry import homography, camera from PCV.localdescriptors import sift def cube_points(c, wid): """ Creates a list of points for plotting a cube with plot. (the first 5 points are the bottom square, some sides repeated). """ p = [] # bottom p.append([c[0]-wid, c[1]-wid, c[2]-wid]) p.append([c[0]-wid, c[1]+wid, c[2]-wid]) p.append([c[0]+wid, c[1]+wid, c[2]-wid]) p.append([c[0]+wid, c[1]-wid, c[2]-wid]) p.append([c[0]-wid, c[1]-wid, c[2]-wid]) #same as first to close plot # top p.append([c[0]-wid, c[1]-wid, c[2]+wid]) p.append([c[0]-wid, c[1]+wid, c[2]+wid]) p.append([c[0]+wid, c[1]+wid, c[2]+wid]) p.append([c[0]+wid, c[1]-wid, c[2]+wid]) p.append([c[0]-wid, c[1]-wid, c[2]+wid]) #same as first to close plot # vertical sides p.append([c[0]-wid, c[1]-wid, c[2]+wid]) p.append([c[0]-wid, c[1]+wid, c[2]+wid]) p.append([c[0]-wid, c[1]+wid, c[2]-wid]) p.append([c[0]+wid, c[1]+wid, c[2]-wid]) p.append([c[0]+wid, c[1]+wid, c[2]+wid]) p.append([c[0]+wid, c[1]-wid, c[2]+wid]) p.append([c[0]+wid, c[1]-wid, c[2]-wid]) return array(p).T def my_calibration(sz): row, col = sz fx = 2555*col/2592 fy = 2586*row/1936 K = diag([fx, fy, 1]) K[0, 2] = 0.5*col K[1, 2] = 0.5*row return K def set_projection_from_camera(K): glMatrixMode(GL_PROJECTION) glLoadIdentity() fx = K[0,0] fy = K[1,1] fovy = 2*math.atan(0.5*height/fy)*180/math.pi aspect = (width*fy)/(height*fx) near = 0.1 far = 100.0 gluPerspective(fovy,aspect,near,far) glViewport(0,0,width,height) def set_modelview_from_camera(Rt): glMatrixMode(GL_MODELVIEW) glLoadIdentity() Rx = np.array([[1,0,0],[0,0,-1],[0,1,0]]) R = Rt[:,:3] U,S,V = np.linalg.svd(R) R = np.dot(U,V) R[0,:] = -R[0,:] t = Rt[:,3] M = np.eye(4) M[:3,:3] = np.dot(R,Rx) M[:3,3] = t M = M.T m = M.flatten() glLoadMatrixf(m) def draw_background(imname): bg_image = pygame.image.load(imname).convert() bg_data = pygame.image.tostring(bg_image,"RGBX",1) glMatrixMode(GL_MODELVIEW) glLoadIdentity() glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) glEnable(GL_TEXTURE_2D) glBindTexture(GL_TEXTURE_2D,glGenTextures(1)) glTexImage2D(GL_TEXTURE_2D,0,GL_RGBA,width,height,0,GL_RGBA,GL_UNSIGNED_BYTE,bg_data) glTexParameterf(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_NEAREST) glTexParameterf(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_NEAREST) glBegin(GL_QUADS) glTexCoord2f(0.0,0.0); glVertex3f(-1.0,-1.0,-1.0) glTexCoord2f(1.0,0.0); glVertex3f( 1.0,-1.0,-1.0) glTexCoord2f(1.0,1.0); glVertex3f( 1.0, 1.0,-1.0) glTexCoord2f(0.0,1.0); glVertex3f(-1.0, 1.0,-1.0) glEnd() glDeleteTextures(1) def draw_teapot(size): glEnable(GL_LIGHTING) glEnable(GL_LIGHT0) glEnable(GL_DEPTH_TEST) glClear(GL_DEPTH_BUFFER_BIT) glMaterialfv(GL_FRONT,GL_AMBIENT,[0,0,0,0]) glMaterialfv(GL_FRONT,GL_DIFFUSE,[0.5,0.0,0.0,0.0]) glMaterialfv(GL_FRONT,GL_SPECULAR,[0.7,0.6,0.6,0.0]) glMaterialf(GL_FRONT,GL_SHININESS,0.25*128.0) glutSolidTeapot(size) width,height = 1000,747 def setup(): pygame.init() pygame.display.set_mode((width,height),OPENGL | DOUBLEBUF) pygame.display.set_caption("OpenGL AR demo") # compute features sift.process_image('book_frontal.JPG', 'im0.sift') l0, d0 = sift.read_features_from_file('im0.sift') sift.process_image('book_perspective.JPG', 'im1.sift') l1, d1 = sift.read_features_from_file('im1.sift') # match features and estimate homography matches = sift.match_twosided(d0, d1) ndx = matches.nonzero()[0] fp = homography.make_homog(l0[ndx, :2].T) ndx2 = [int(matches[i]) for i in ndx] tp = homography.make_homog(l1[ndx2, :2].T) model = homography.RansacModel() H, inliers = homography.H_from_ransac(fp, tp, model) K = my_calibration((747, 1000)) cam1 = camera.Camera(hstack((K, dot(K, array([[0], [0], [-1]]))))) box = cube_points([0, 0, 0.1], 0.1) box_cam1 = cam1.project(homography.make_homog(box[:, :5])) box_trans = homography.normalize(dot(H,box_cam1)) cam2 = camera.Camera(dot(H, cam1.P)) A = dot(linalg.inv(K), cam2.P[:, :3]) A = array([A[:, 0], A[:, 1], cross(A[:, 0], A[:, 1])]).T cam2.P[:, :3] = dot(K, A) Rt=dot(linalg.inv(K),cam2.P) setup() draw_background("book_perspective.bmp") set_projection_from_camera(K) set_modelview_from_camera(Rt) draw_teapot(0.05) pygame.display.flip() while True: for event in pygame.event.get(): if event.type==pygame.QUIT: sys.exit() import math import pickle from pylab import * from OpenGL.GL import * from OpenGL.GLU import * from OpenGL.GLUT import * import pygame, pygame.image from pygame.locals import * from PCV.geometry import homography, camera from PCV.localdescriptors import sift def cube_points(c, wid): """ Creates a list of points for plotting a cube with plot. (the first 5 points are the bottom square, some sides repeated). """ p = [] # bottom p.append([c[0]-wid, c[1]-wid, c[2]-wid]) p.append([c[0]-wid, c[1]+wid, c[2]-wid]) p.append([c[0]+wid, c[1]+wid, c[2]-wid]) p.append([c[0]+wid, c[1]-wid, c[2]-wid]) p.append([c[0]-wid, c[1]-wid, c[2]-wid]) #same as first to close plot # top p.append([c[0]-wid, c[1]-wid, c[2]+wid]) p.append([c[0]-wid, c[1]+wid, c[2]+wid]) p.append([c[0]+wid, c[1]+wid, c[2]+wid]) p.append([c[0]+wid, c[1]-wid, c[2]+wid]) p.append([c[0]-wid, c[1]-wid, c[2]+wid]) #same as first to close plot # vertical sides p.append([c[0]-wid, c[1]-wid, c[2]+wid]) p.append([c[0]-wid, c[1]+wid, c[2]+wid]) p.append([c[0]-wid, c[1]+wid, c[2]-wid]) p.append([c[0]+wid, c[1]+wid, c[2]-wid]) p.append([c[0]+wid, c[1]+wid, c[2]+wid]) p.append([c[0]+wid, c[1]-wid, c[2]+wid]) p.append([c[0]+wid, c[1]-wid, c[2]-wid]) return array(p).T def my_calibration(sz): row, col = sz fx = 2555*col/2592 fy = 2586*row/1936 K = diag([fx, fy, 1]) K[0, 2] = 0.5*col K[1, 2] = 0.5*row return K def set_projection_from_camera(K): glMatrixMode(GL_PROJECTION) glLoadIdentity() fx = K[0,0] fy = K[1,1] fovy = 2*math.atan(0.5*height/fy)*180/math.pi aspect = (width*fy)/(height*fx) near = 0.1 far = 100.0 gluPerspective(fovy,aspect,near,far) glViewport(0,0,width,height) def set_modelview_from_camera(Rt): glMatrixMode(GL_MODELVIEW) glLoadIdentity() Rx = np.array([[1,0,0],[0,0,-1],[0,1,0]]) R = Rt[:,:3] U,S,V = np.linalg.svd(R) R = np.dot(U,V) R[0,:] = -R[0,:] t = Rt[:,3] M = np.eye(4) M[:3,:3] = np.dot(R,Rx) M[:3,3] = t M = M.T m = M.flatten() glLoadMatrixf(m) def draw_background(imname): bg_image = pygame.image.load(imname).convert() bg_data = pygame.image.tostring(bg_image,"RGBX",1) glMatrixMode(GL_MODELVIEW) glLoadIdentity() glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) glEnable(GL_TEXTURE_2D) glBindTexture(GL_TEXTURE_2D,glGenTextures(1)) glTexImage2D(GL_TEXTURE_2D,0,GL_RGBA,width,height,0,GL_RGBA,GL_UNSIGNED_BYTE,bg_data) glTexParameterf(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_NEAREST) glTexParameterf(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_NEAREST) glBegin(GL_QUADS) glTexCoord2f(0.0,0.0); glVertex3f(-1.0,-1.0,-1.0) glTexCoord2f(1.0,0.0); glVertex3f( 1.0,-1.0,-1.0) glTexCoord2f(1.0,1.0); glVertex3f( 1.0, 1.0,-1.0) glTexCoord2f(0.0,1.0); glVertex3f(-1.0, 1.0,-1.0) glEnd() glDeleteTextures(1) def draw_teapot(size): glEnable(GL_LIGHTING) glEnable(GL_LIGHT0) glEnable(GL_DEPTH_TEST) glClear(GL_DEPTH_BUFFER_BIT) glMaterialfv(GL_FRONT,GL_AMBIENT,[0,0,0,0]) glMaterialfv(GL_FRONT,GL_DIFFUSE,[0.5,0.0,0.0,0.0]) glMaterialfv(GL_FRONT,GL_SPECULAR,[0.7,0.6,0.6,0.0]) glMaterialf(GL_FRONT,GL_SHININESS,0.25*128.0) glutSolidTeapot(size) width,height = 1000,747 def setup(): pygame.init() pygame.display.set_mode((width,height),OPENGL | DOUBLEBUF) pygame.display.set_caption("OpenGL AR demo") # compute features sift.process_image('book_frontal.JPG', 'im0.sift') l0, d0 = sift.read_features_from_file('im0.sift') sift.process_image('book_perspective.JPG', 'im1.sift') l1, d1 = sift.read_features_from_file('im1.sift') # match features and estimate homography matches = sift.match_twosided(d0, d1) ndx = matches.nonzero()[0] fp = homography.make_homog(l0[ndx, :2].T) ndx2 = [int(matches[i]) for i in ndx] tp = homography.make_homog(l1[ndx2, :2].T) model = homography.RansacModel() H, inliers = homography.H_from_ransac(fp, tp, model) K = my_calibration((747, 1000)) cam1 = camera.Camera(hstack((K, dot(K, array([[0], [0], [-1]]))))) box = cube_points([0, 0, 0.1], 0.1) box_cam1 = cam1.project(homography.make_homog(box[:, :5])) box_trans = homography.normalize(dot(H,box_cam1)) cam2 = camera.Camera(dot(H, cam1.P)) A = dot(linalg.inv(K), cam2.P[:, :3]) A = array([A[:, 0], A[:, 1], cross(A[:, 0], A[:, 1])]).T cam2.P[:, :3] = dot(K, A) Rt=dot(linalg.inv(K),cam2.P) setup() draw_background("book_perspective.bmp") set_projection_from_camera(K) set_modelview_from_camera(Rt) draw_teapot(0.05) pygame.display.flip() while True: for event in pygame.event.get(): if event.type==pygame.QUIT: sys.exit()
(二)动态
下图使用手机拍摄的gif,用的model是fox。
图三
代码链接:https://download.csdn.net/download/weixin_43842653/11094470
三、实验中遇到的问题
1.下载OpenGL
链接:https://www.lfd.uci.edu/~gohlke/pythonlibs/#pyopengl
ctrl+F搜索PyOpenGL,选择与自己python版本相应的版本下载。
2.实现动态增强现实时出现下图问题:
.
说明你的图书封面拍摄有问题,可以换成横着拍(或竖着排)如:
右图就无法实现将fox放到书上的效果,换成横着拍的左图即可。