摄像头替代眼动仪

从dlib库中发现，有人脸检测相关的api函数，其中为68个标记点。但是无法精确定位瞳孔，甚至黑眼球。顶多粗略检测眼睛框（还是散点）。所以秦学英老师给了个PLFD论文，通过multi-view information角度出发，在预测68点的同时，还计算量人脸的三维信息：yaw，roll，pitch三个角度。

说干就干，那么怎么求得yaw，roll，pitch三个角度呢，俗话说计算视觉大法好，一个pnp函数解决，当然这里需要极强的功底，我是在增强现实课程中学的，头早秃了，用二维点算三维信息，想想都牛啊。

    def return_pitch_yaw_roll(self, image, radians=False):
    
         #The dlib shape predictor returns 68 points, we are interested only in a few of those
         # TRACKED_POINTS = (0, 4, 8, 12, 16, 17, 26, 27, 30, 33, 36, 39, 42, 45, 62)
         TRACKED_POINTS = [17, 21, 22, 26, 36, 39, 42, 45, 31, 35, 48, 54, 57, 8]

       
         LEFT_EYEBROW_LEFT = [6.825897, 6.760612, 4.402142]
         LEFT_EYEBROW_RIGHT = [1.330353, 7.122144, 6.903745]
         RIGHT_EYEBROW_LEFT = [-1.330353, 7.122144, 6.903745]
         RIGHT_EYEBROW_RIGHT = [-6.825897, 6.760612, 4.402142]
         LEFT_EYE_LEFT = [5.311432, 5.485328, 3.987654]
         LEFT_EYE_RIGHT = [1.789930, 5.393625, 4.413414]
         RIGHT_EYE_LEFT = [-1.789930, 5.393625, 4.413414]
         RIGHT_EYE_RIGHT = [-5.311432, 5.485328, 3.987654]
         NOSE_LEFT = [2.005628, 1.409845, 6.165652]
         NOSE_RIGHT = [-2.005628, 1.409845, 6.165652]
         MOUTH_LEFT = [2.774015, -2.080775, 5.048531]
         MOUTH_RIGHT = [-2.774015, -2.080775, 5.048531]
         LOWER_LIP = [0.000000, -3.116408, 6.097667]
         CHIN = [0.000000, -7.415691, 4.070434]
         landmarks_3D = np.float32([LEFT_EYEBROW_LEFT,
                                    LEFT_EYEBROW_RIGHT,
                                    RIGHT_EYEBROW_LEFT,
                                    RIGHT_EYEBROW_RIGHT,
                                    LEFT_EYE_LEFT,
                                    LEFT_EYE_RIGHT,
                                    RIGHT_EYEBROW_LEFT,
                                    RIGHT_EYEBROW_RIGHT,
                                    NOSE_LEFT,
                                    NOSE_RIGHT,
                                    MOUTH_LEFT,
                                    MOUTH_RIGHT,
                                    LOWER_LIP,
                                    CHIN])
         #Return the 2D position of our landmarks
         landmarks_2D = self._return_landmarks(inputImg=image, points_to_return=TRACKED_POINTS)
         if landmarks_2D is not None :
             #Print som red dots on the image
             #for point in landmarks_2D:
                 #cv2.circle(frame,( point[0], point[1] ), 2, (0,0,255), -1)


             #Applying the PnP solver to find the 3D pose
             #of the head from the 2D position of the
             #landmarks.
             #retval - bool
             #rvec - Output rotation vector that, together with tvec, brings
             #points from the world coordinate system to the camera coordinate system.
             #tvec - Output translation vector. It is the position of the world origin (SELLION) in camera co-ords
             retval, rvec, tvec = cv2.solvePnP(landmarks_3D,
                                               landmarks_2D,
                                               self.camera_matrix,
                                               self.camera_distortion)
             #Get as input the rotational vector
             #Return a rotational matrix
             rmat, _ = cv2.Rodrigues(rvec)
             pose_mat = cv2.hconcat((rmat,tvec))
             #euler_angles contain (pitch, yaw, roll)
             # euler_angles = cv2.DecomposeProjectionMatrix(projMatrix=rmat, cameraMatrix=self.camera_matrix, rotMatrix, transVect, rotMatrX=None, rotMatrY=None, rotMatrZ=None)
             _, _, _, _, _, _,euler_angles = cv2.decomposeProjectionMatrix(pose_mat)
             return list(euler_angles)

             head_pose = [ rmat[0,0], rmat[0,1], rmat[0,2], tvec[0],
                           rmat[1,0], rmat[1,1], rmat[1,2], tvec[1],
                           rmat[2,0], rmat[2,1], rmat[2,2], tvec[2],
                                 0.0,      0.0,        0.0,    1.0 ]
             #print(head_pose) #TODO remove this line
             return self.rotationMatrixToEulerAngles(rmat)
         else:return None



    # Calculates rotation matrix to euler angles
    # The result is the same as MATLAB except the order
    # of the euler angles ( x and z are swapped ).
    def rotationMatrixToEulerAngles(self, R) :

        #assert(isRotationMatrix(R))

        #To prevent the Gimbal Lock it is possible to use
        #a threshold of 1e-6 for discrimination
        sy = math.sqrt(R[0,0] * R[0,0] +  R[1,0] * R[1,0])
        singular = sy < 1e-6

        if  not singular :
            x = math.atan2(R[2,1] , R[2,2])
            y = math.atan2(-R[2,0], sy)
            z = math.atan2(R[1,0], R[0,0])
        else :
            x = math.atan2(-R[1,2], R[1,1])
            y = math.atan2(-R[2,0], sy)
            z = 0

        return np.array([x, y, z])

问题解决，又出现问题，论文提出的方法，跑起来贼慢，运行时候还贼卡，也没见得多精确，且似乎与dlib不能很好地兼容。改！于是用了dlib训练好的模型，于是根据68点规则，选取了里面的眼睛部分的点，把他标注了出来，当然，顺便把鼻子，耳朵，嘴巴和眨眼次数看起来牛的功能顺手用68点信息做了。

# coordinates to compute the eye aspect ratio for both eyes
		leftEye = shape[lStart:lEnd]
		rightEye = shape[rStart:rEnd]
		leftEAR = eye_aspect_ratio(leftEye)
		rightEAR = eye_aspect_ratio(rightEye)

然后也就初步成型了，剩下的就是利用图像处理的方法，去想办法精确到瞳孔了，当然先是到黑眼球，在瞳孔，方法是可以想到，但是实现是有够头疼的。