红外相机和RGB相机标定：实现两种模态数据融合

1. 前期准备

1. RGB相机：森云智能SG2-IMX390，1个
2. 红外相机：艾睿光电IR-Pilot 640X-32G，1个
3. 红外标定板：https://item.taobao.com/item.htm?_u=jp3fdd12b99&id=644506141871&spm=a1z09.2.0.0.5f822e8dKrxxYI

2.操作步骤

2.1 采集标定数据

两种模态相机均未进行内参标定，如果发现原始图片畸变较大，可以先进行内参标定。数据采集代码如下，加热红外标定板后断电，移动标定板到合适的位置，按下s键，同时保存IR图和RG图

#!/usr/bin/env python3
import cv2 , time
import numpy as np

ir_dev = "/dev/video6"
rgb_dev = "/dev/video0"
# define a video capture object 
ir_vid = cv2.VideoCapture(ir_dev) 
rgb_vid = cv2.VideoCapture(rgb_dev) 

count = 0
while(True): 
      
    # Capture the video frame by frame 
    st_time = time.time()
    ret, ir_frame = ir_vid.read()
    # print(f"{time.time() - st_time}") 
    ret, rgb_frame = rgb_vid.read()
    print(f"{time.time() - st_time}") 
    
    # Display the resulting frame 
    height, width = ir_frame.shape[:2]
    #(512,1280)
    index = [2*i+1 for i in range(width//2)]
    vis_ir_frame = ir_frame[:,index,:]

    vis_rgb_frame = cv2.resize(rgb_frame, (640,512))
    cv2.imshow('IR frame', vis_ir_frame) 
    cv2.imshow('RGB frame', vis_rgb_frame) 

    key = cv2.waitKey(1) & 0xFF 
    if key == ord('q'): 
        break
    if key == ord('s'):
        cv2.imwrite(f"IR_{count}.png", vis_ir_frame)
        cv2.imwrite(f"RGB_{count}.png", vis_rgb_frame)
        count += 1
  
# After the loop release the cap object 
ir_vid.release() 
rgb_vid.release() 
# Destroy all the windows 
cv2.destroyAllWindows() 

2.2 进行标定

核心操作是调用opencv函数cv2.findHomography计算两个相机之间的单应性矩阵，代码如下

#!/usr/bin/python
# -*- coding: UTF-8 -*-
import cv2
import numpy as np

def find_chessboard(filename, pattern=(9,8), wind_name="rgb"):
    # read input image
    img = cv2.imread(filename)
    # cv2.imshow("raw", img)
    # img = cv2.undistort(img, camera_matrix, distortion_coefficients)

    # convert the input image to a grayscale
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    # Find the chess board corners
    ret, corners = cv2.findChessboardCorners(gray, pattern, None)
    
    # if chessboard corners are detected
    if ret == True:
        # Draw and display the corners
        img = cv2.drawChessboardCorners(img, pattern, corners, ret)
        
        #Draw number，打印角点编号，便于确定对应点
        corners = np.ceil(corners[:,0,:])
        for i, pt in enumerate(corners): 
            cv2.putText(img, str(i), (int(pt[0]),int(pt[1])), cv2.FONT_HERSHEY_COMPLEX, 0.3, (0,255,0), 1)
        cv2.imshow(wind_name,img)

        return corners
    
    return None
        

if __name__ == '__main__' :
    idx = 2 #0~71
    rgb_img = cv2.imread(f"RGB_{idx}.png")
    t_img = cv2.imread(f"IR_{idx}.png")

    #chessboard grid nums in rgb ,注意观察，同一块标定板在RGB相机和红外相机中的格子说可能不一样
    rgb_width, rgb_height = 9, 8
    rgb_corners = find_chessboard(f"RGB_{idx}.png", (rgb_width, rgb_height), "rgb")

    #chessboard grid nums in thermal 
    thermal_width, thermal_height = 11, 8
    t_corners = find_chessboard(f"IR_{idx}.png", (thermal_width, thermal_height), "thermal")
    
    if rgb_corners is not None and t_corners is not None:
        # test the id correspondence between rgb and thermal corners
        rgb_idx = 27 #可视化一个点，确认取对应点的过程是否正确
        row, col = rgb_idx//rgb_width, rgb_idx%rgb_width
        t_idx = row*thermal_width + col + 1

        pt = rgb_corners[rgb_idx]
        cv2.putText(rgb_img, str(rgb_idx), (int(pt[0]),int(pt[1])), cv2.FONT_HERSHEY_COMPLEX, 0.3, (0,255,0), 1)
        pt = t_corners[t_idx]
        cv2.putText(t_img, str(t_idx), (int(pt[0]),int(pt[1])), cv2.FONT_HERSHEY_COMPLEX, 0.3, (0,255,0), 1)
        cv2.imshow(f"Point {rgb_idx} on rgb", rgb_img)
        cv2.imshow(f"Point {t_idx} on thermal", t_img)


        # Calculate Homography
        src_pts = []
        for rgb_idx in range(len(rgb_corners)):
            row, col = rgb_idx//9, rgb_idx%9
            t_idx = row*11+col+1
            src_pts.append(t_corners[t_idx])
        h, status = cv2.findHomography(np.array(src_pts)[:,None,:], rgb_corners[:,None,:])
        
        np.savetxt("calib.param", h)
    
        # Warp source image to destination based on homography
        t_warp = cv2.warpPerspective(t_img, h, (640,512), borderValue=(255,255,255))

        #colorize
        t_warp = cv2.applyColorMap(t_warp, cv2.COLORMAP_JET)

        #mix rgb and thermal
        alpha = 0.5
        merge = cv2.addWeighted(rgb_img, alpha, t_warp, 1-alpha, gamma=0)

        cv2.imshow("warp", merge)

    cv2.waitKey(0)
    cv2.destroyAllWindows()

运行结果如下，观察红外和RGB图中角点的对应关系，编号已经可视化出来了

同时，也单独画出了1个对应后的点，如下图，可检查映射关系是否找对

最后，融合结果如下图所示：