Dataset: https://personalpages.manchester.ac.uk/staff/timothy.f.cootes/data/talking_face/talking_face.html
step 1: load images and points
import cv2
from matplotlib import pyplot as plt
import os
import numpy as np
%matplotlib inline
random_ind = np.random.randint(0, 4999, size=(200,))
images_dir = "./Talking Face Video/images"
images = []
for i in random_ind:
file_name = "franck_"+"{:05d}".format(i)+".jpg"
images.append(cv2.cvtColor(cv2.imread(os.path.join(images_dir, file_name)), cv2.COLOR_BGR2GRAY))
points_dir = "./Talking Face Video/points"
points_images = []
for i in random_ind:
file_name = "franck_"+"{:05d}".format(i)+".pts"
with open(os.path.join(points_dir, file_name),'r') as f:
lines = f.readlines()
points_images.append( [tuple([int(float(num)) for num in line.split()]) for line in lines[3:71]])
step 2: select the ROI of faces and resize it, then adjust the points coordinate
faceCascade = cv2.CascadeClassifier("/usr/local/Cellar/opencv/4.0.1/share/opencv4/haarcascades/haarcascade_frontalface_default.xml")
faces = []
faces_resized = []
points_faces = []
points_faces_resized = []
for image, points_image in zip(images, points_images):
fcs = faceCascade.detectMultiScale(
image,
scaleFactor=1.1,
minNeighbors=5,
minSize=(30, 30),
flags = 0
)
for (x, y, w, h) in fcs:
fcs_img = image[y:y+h+70, x:x+w]
points_face = [tuple(np.subtract(pt, (x,y))) for pt in points_image]
faces.append(fcs_img)
points_faces.append(points_face)
fcs_img_resized = cv2.resize(fcs_img, (350,400))
points_face_resized = [( int(pt[0]* float(350/fcs_img.shape[1])), int(pt[1]* float(400/fcs_img.shape[0])) ) for pt in points_face]
faces_resized.append(fcs_img_resized)
points_faces_resized.append(points_face_resized)
def plot_images(images, points, rows=11, cols=15, size=(10,12)):
plt.figure(figsize=size)
for i in range(rows * cols):
plt.subplot(rows, cols, i + 1)
image = cv2.cvtColor(images[i].copy(), cv2.COLOR_GRAY2BGR)
ind = 0
for point in points[i]:
cv2.circle(image, point, 2, (255,0,0), -1)
cv2.putText(image, str(ind), tuple(np.add(point, (5,0))), cv2.FONT_HERSHEY_SIMPLEX, 0.3, (0,255,0), 1, cv2.LINE_AA)
ind = ind+1
plt.imshow(image, cmap=plt.cm.gray)
#plt.xticks(())
#plt.yticks(())
plot_images([faces[0], faces_resized[0]], [points_faces[0], points_faces_resized[0]], rows=1, cols=2, size=(10,15))
step 3: calculate the mean shape
mean_shape_points = np.zeros(np.array(points_faces_resized[0]).shape, dtype='float')
for points_face_resized in points_faces_resized:
mean_shape_points = mean_shape_points + np.array(points_face_resized)
mean_shape_points = (mean_shape_points/len(points_faces_resized)).astype('int32')
mean_shape_points = [tuple(i) for i in mean_shape_points]
mean_shape_gray = np.zeros(faces_resized[0].shape, dtype='uint8')
ind = 0
for point in mean_shape_points:
cv2.circle(mean_shape_gray, point, 2, (255,255,255), 2)
#cv2.putText(mean_shape, str(ind), tuple(np.add(tuple(point), (5,0))), cv2.FONT_HERSHEY_SIMPLEX, 0.3, (255,255,255), 1, cv2.LINE_AA)
ind = ind+1
plt.imshow(mean_shape_gray, cmap=plt.cm.gray)
<matplotlib.image.AxesImage at 0x124c8e518>
step 4: triangulation resized faces and the mean shape
face = cv2.cvtColor(faces_resized[0].copy(), cv2.COLOR_GRAY2BGR)
ind = 0
for point in points_faces_resized[0]:
cv2.circle(face, point, 2, (255,0,0), -1)
cv2.putText(face, str(ind), tuple(np.add(point, (5,0))), cv2.FONT_HERSHEY_SIMPLEX, 0.3, (0,255,0), 1, cv2.LINE_AA)
ind = ind+1
plt.imshow(face)
<matplotlib.image.AxesImage at 0x12e4ac080>
a. get face area in convex
face_gray = faces_resized[0].copy()
convexhull = cv2.convexHull(np.array(points_faces_resized[0]))
mask = np.zeros_like(face_gray)
mask = cv2.fillConvexPoly(mask, convexhull, 255)
face_area = cv2.bitwise_and(face_gray, face_gray, mask=mask)
plt.imshow(face_area, cmap=plt.cm.gray)
<matplotlib.image.AxesImage at 0x1218736d8>
b. triangulation of face area
face = cv2.cvtColor(face_area, cv2.COLOR_GRAY2BGR)
points_inds = {points_faces_resized[0][i]:i for i in range(len(points_faces_resized[0]))}
rect = cv2.boundingRect(convexhull)
subdiv = cv2.Subdiv2D(rect)
subdiv.insert(points_faces_resized[0])
triangles = subdiv.getTriangleList().astype('int')
triangles_inds = []
for triangle in triangles:
pt1 = (triangle[0], triangle[1])
pt2 = (triangle[2], triangle[3])
pt3 = (triangle[4], triangle[5])
cv2.line(face, pt1, pt2, (0,0,255),1)
cv2.line(face, pt2, pt3, (0,0,255),1)
cv2.line(face, pt3, pt1, (0,0,255),1)
triangles_inds.append([points_inds[pt1], points_inds[pt2], points_inds[pt3]])
plt.imshow(face)
<matplotlib.image.AxesImage at 0x121857438>
c. triangulation of the mean shape
mean_shape_points = [(int(i[0]), int(i[1]))for i in mean_shape_points]
points_inds_mean_shape = {i:mean_shape_points[i] for i in range(len(mean_shape_points))}
convexhull = cv2.convexHull(np.array(mean_shape_points))
mask_mean_shape = np.zeros_like(mean_shape_gray)
mask_mean_shape = cv2.cvtColor(mask_mean_shape, cv2.COLOR_GRAY2BGR)
mask_mean_shape = cv2.fillConvexPoly(mask_mean_shape, convexhull, (255,255,255))
for triangle_inds in triangles_inds:
pt1 = points_inds_mean_shape[triangle_inds[0]]
pt2 = points_inds_mean_shape[triangle_inds[1]]
pt3 = points_inds_mean_shape[triangle_inds[2]]
cv2.line(mask_mean_shape, pt1, pt2, (0,0,255),1)
cv2.line(mask_mean_shape, pt2, pt3, (0,0,255),1)
cv2.line(mask_mean_shape, pt3, pt1, (0,0,255),1)
plt.imshow(mask_mean_shape)
<matplotlib.image.AxesImage at 0x124d9de80>
step 5: swap face to the mean shape
plot_images([cv2.cvtColor(face,cv2.COLOR_BGR2GRAY), cv2.cvtColor(mask_mean_shape,cv2.COLOR_BGR2GRAY)], [points_faces_resized[0], mean_shape_points], rows=1, cols=2, size=(10,15))
a. convert white pixels in the mean shape to black pixels
for i in range(mask_mean_shape.shape[0]):
for j in range(mask_mean_shape.shape[1]):
if (mask_mean_shape[i][j] == [255,255,255]).all():
mask_mean_shape[i][j] = np.array([0,0,0])
plt.imshow(mask_mean_shape)
<matplotlib.image.AxesImage at 0x124e05b00>
b. swap face patches to the mean shape
points_inds = {v:k for k,v in points_inds.items()}
for triangle_inds in triangles_inds:
# calculate the new coordinates of points in each face patch
points_face = np.array([points_inds[triangle_inds[0]],
points_inds[triangle_inds[1]],
points_inds[triangle_inds[2]]])
points_mean_shape = np.array([points_inds_mean_shape[triangle_inds[0]],
points_inds_mean_shape[triangle_inds[1]],
points_inds_mean_shape[triangle_inds[2]]])
# crop the face path in the original face
convexhull = cv2.convexHull(points_face)
rect = cv2.boundingRect(convexhull)
(x, y, w, h) = rect
cropped_triangle = face[y:y+h, x:x+w]
cropped_triangle_mask = np.zeros((h,w), np.uint8)
points_face_triangle = np.array([[points_face[0][0]-x, points_face[0][1]-y],
[points_face[1][0]-x, points_face[1][1]-y],
[points_face[2][0]-x, points_face[2][1]-y]])
cropped_triangle_mask = cv2.fillConvexPoly(cropped_triangle_mask, points_face_triangle, 255)
#plt.subplot(141)
#plt.imshow(cropped_triangle_mask, cmap=plt.cm.gray)
cropped_triangle = cv2.bitwise_and(cropped_triangle, cropped_triangle, mask=cropped_triangle_mask)
#plt.subplot(142)
#plt.imshow(cropped_triangle)
# warp the face patch in the original face to the coresponding patch in the mean shape, using the Affine Transformation
convexhull = cv2.convexHull(points_mean_shape)
rect = cv2.boundingRect(convexhull)
(x, y, w, h) = rect
points_mean_shape_triangle = np.array([[points_mean_shape[0][0]-x, points_mean_shape[0][1]-y],
[points_mean_shape[1][0]-x, points_mean_shape[1][1]-y],
[points_mean_shape[2][0]-x, points_mean_shape[2][1]-y]])
M = cv2.getAffineTransform(np.float32(points_face_triangle), np.float32(points_mean_shape_triangle))
warped_face_triangle = cv2.warpAffine(cropped_triangle, M, (w,h))
#plt.subplot(143)
#plt.imshow(warped_face_triangle)
# paste the warped face patch to the mean shape
triangle_rect_mean_shape = mask_mean_shape[y:y+h, x:x+w]
triangle_rect_mean_shape = cv2.add(triangle_rect_mean_shape, warped_face_triangle)
mask_mean_shape[y:y+h, x:x+w] = triangle_rect_mean_shape
#plt.subplot(144)
#plt.imshow(mask_mean_shape)
#break
step 6: results
plt.subplot(121)
plt.imshow(face)
plt.title("the original face")
plt.subplot(122)
plt.imshow(mask_mean_shape)
plt.title("face swaped to the mean shape")
Text(0.5, 1.0, 'face swaped to the mean shape')
plt.imshow(cv2.subtract(face, mask_mean_shape))
plt.title("differences between the original face and the swaped face")
Text(0.5, 1.0, 'differences between the original face and the swaped face')