import cv2 import copy import numpy as np import open3d as o3d import mediapipe as mp mp_drawing = mp.solutions.drawing_utils mp_drawing_styles = mp.solutions.drawing_styles mp_holistic = mp.solutions.holistic from scipy.spatial.transform import Rotation as Rtool def process_landmark(face_landmarks, width, height): landmark1 = face_landmarks.landmark ## 478 is known number np_array = np.zeros((len(landmark1), 3), np.float64) for i in range(len(landmark1)): np_array[i, 0] = landmark1[i].x np_array[i, 1] = landmark1[i].y np_array[i, 2] = landmark1[i].z np_array[:, 0] = np_array[:, 0] * width np_array[:, 1] = - np_array[:, 1] * height np_array[:, 2] = - np_array[:, 2] * width return np_array def process_face_landmark(face_landmarks, width, height): landmark_3d = process_landmark(face_landmarks, width, height) x_axis = landmark_3d[280] - landmark_3d[50] x_axis += landmark_3d[352] - landmark_3d[123] x_axis += landmark_3d[280] - landmark_3d[50] x_axis += landmark_3d[376] - landmark_3d[147] x_axis += landmark_3d[416] - landmark_3d[192] x_axis += landmark_3d[298] - landmark_3d[68] x_axis += landmark_3d[301] - landmark_3d[71] y_axis = landmark_3d[10] - landmark_3d[152] y_axis += landmark_3d[151] - landmark_3d[152] y_axis += landmark_3d[8] - landmark_3d[17] y_axis += landmark_3d[5] - landmark_3d[200] y_axis += landmark_3d[6] - landmark_3d[199] y_axis += landmark_3d[8] - landmark_3d[18] y_axis += landmark_3d[9] - landmark_3d[175] x_axis /= np.linalg.norm(x_axis) y_axis /= np.linalg.norm(y_axis) z_axis = np.cross(x_axis, y_axis) z_axis /= np.linalg.norm(z_axis) y_axis = np.cross(z_axis, x_axis) matrix = Rtool.from_matrix(np.transpose(np.array([x_axis, y_axis, z_axis]))) * Rtool.from_rotvec([-0.25, 0, 0]) rotvec = matrix.as_rotvec() return rotvec def process_head_roation(mpeg4file): cap = cv2.VideoCapture(mpeg4file) rots_head = [] with mp_holistic.Holistic(min_detection_confidence=0.2, min_tracking_confidence=0.2) as holistic: while cap.isOpened(): success, image = cap.read() if not success: break height, width = image.shape[0], image.shape[1] image.flags.writeable = False image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) results = holistic.process(image) # Draw landmark annotation on the image. # mp_drawing.draw_landmarks(image, results.face_landmarks, mp_holistic.FACEMESH_CONTOURS, landmark_drawing_spec=None, # connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_contours_style()) # mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS, # landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style()) # mp_drawing.draw_landmarks(image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS, # mp_drawing_styles.get_default_hand_landmarks_style(), mp_drawing_styles.get_default_hand_connections_style()) # mp_drawing.draw_landmarks(image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS, # mp_drawing_styles.get_default_hand_landmarks_style(), mp_drawing_styles.get_default_hand_connections_style()) pose_landmarks = process_landmark(results.pose_landmarks) left_hand_landmarks = process_landmark(results.left_hand_landmarks) right_hand_landmarks = process_landmark(results.right_hand_landmarks) rotvec = process_face_landmark(results.face_landmarks, width, height) rots_head.append(rotvec.copy()) return rots_head if __name__=="__main__": name_ = "0.mp4" process_head_roation(name_)