Face Swapping

Dataset: https://personalpages.manchester.ac.uk/staff/timothy.f.cootes/data/talking_face/talking_face.html

step 1: load images and points

import cv2
from matplotlib import pyplot as plt
import os
import numpy as np
%matplotlib inline

random_ind = np.random.randint(0, 4999, size=(200,))

images_dir = "./Talking Face Video/images"

images = []
for i in random_ind:
    file_name = "franck_"+"{:05d}".format(i)+".jpg"
    
    images.append(cv2.cvtColor(cv2.imread(os.path.join(images_dir, file_name)), cv2.COLOR_BGR2GRAY))

points_dir = "./Talking Face Video/points"


points_images = []
for i in random_ind:
    file_name = "franck_"+"{:05d}".format(i)+".pts"
    
    with open(os.path.join(points_dir, file_name),'r') as f:
        lines = f.readlines()
        points_images.append( [tuple([int(float(num)) for num in line.split()]) for line in lines[3:71]])

step 2: select the ROI of faces and resize it, then adjust the points coordinate

faceCascade = cv2.CascadeClassifier("/usr/local/Cellar/opencv/4.0.1/share/opencv4/haarcascades/haarcascade_frontalface_default.xml")

faces = []
faces_resized = []
points_faces = []
points_faces_resized = []

for image, points_image in zip(images, points_images):
    fcs = faceCascade.detectMultiScale(
            image,
            scaleFactor=1.1,
            minNeighbors=5,
            minSize=(30, 30),
            flags = 0
        )
    for (x, y, w, h) in fcs:
        fcs_img = image[y:y+h+70, x:x+w]
        points_face = [tuple(np.subtract(pt, (x,y))) for pt in points_image]
        
        faces.append(fcs_img)
        points_faces.append(points_face)
        
        fcs_img_resized = cv2.resize(fcs_img, (350,400))
        points_face_resized = [( int(pt[0]* float(350/fcs_img.shape[1])), int(pt[1]* float(400/fcs_img.shape[0])) ) for pt in points_face]
        
        faces_resized.append(fcs_img_resized)
        points_faces_resized.append(points_face_resized)

def plot_images(images, points, rows=11, cols=15, size=(10,12)):
    plt.figure(figsize=size)
    for i in range(rows * cols):
        plt.subplot(rows, cols, i + 1)
        
        image = cv2.cvtColor(images[i].copy(), cv2.COLOR_GRAY2BGR)
        ind = 0
        for point in points[i]:
            cv2.circle(image, point, 2, (255,0,0), -1)
            cv2.putText(image, str(ind), tuple(np.add(point, (5,0))), cv2.FONT_HERSHEY_SIMPLEX, 0.3, (0,255,0), 1, cv2.LINE_AA)
            ind = ind+1

        plt.imshow(image, cmap=plt.cm.gray)
        #plt.xticks(())
        #plt.yticks(())

plot_images([faces[0], faces_resized[0]], [points_faces[0], points_faces_resized[0]], rows=1, cols=2, size=(10,15))

step 3: calculate the mean shape

mean_shape_points = np.zeros(np.array(points_faces_resized[0]).shape, dtype='float')

for points_face_resized in points_faces_resized:
    mean_shape_points = mean_shape_points + np.array(points_face_resized)
    
mean_shape_points = (mean_shape_points/len(points_faces_resized)).astype('int32')
mean_shape_points = [tuple(i) for i in mean_shape_points]

mean_shape_gray = np.zeros(faces_resized[0].shape, dtype='uint8')

ind = 0
for point in mean_shape_points:
    cv2.circle(mean_shape_gray, point, 2, (255,255,255), 2)
    #cv2.putText(mean_shape, str(ind), tuple(np.add(tuple(point), (5,0))), cv2.FONT_HERSHEY_SIMPLEX, 0.3, (255,255,255), 1, cv2.LINE_AA)
    ind = ind+1

plt.imshow(mean_shape_gray, cmap=plt.cm.gray)

<matplotlib.image.AxesImage at 0x124c8e518>

step 4: triangulation resized faces and the mean shape

face = cv2.cvtColor(faces_resized[0].copy(), cv2.COLOR_GRAY2BGR)

ind = 0
for point in points_faces_resized[0]:
    cv2.circle(face, point, 2, (255,0,0), -1)
    cv2.putText(face, str(ind), tuple(np.add(point, (5,0))), cv2.FONT_HERSHEY_SIMPLEX, 0.3, (0,255,0), 1, cv2.LINE_AA)
    ind = ind+1
    
plt.imshow(face)

<matplotlib.image.AxesImage at 0x12e4ac080>

a. get face area in convex

face_gray = faces_resized[0].copy()
convexhull = cv2.convexHull(np.array(points_faces_resized[0]))
mask = np.zeros_like(face_gray)
mask = cv2.fillConvexPoly(mask, convexhull, 255)
face_area = cv2.bitwise_and(face_gray, face_gray, mask=mask)

plt.imshow(face_area, cmap=plt.cm.gray)

<matplotlib.image.AxesImage at 0x1218736d8>

b. triangulation of face area

face = cv2.cvtColor(face_area, cv2.COLOR_GRAY2BGR)

points_inds = {points_faces_resized[0][i]:i for i in range(len(points_faces_resized[0]))}

rect = cv2.boundingRect(convexhull)
subdiv = cv2.Subdiv2D(rect)
subdiv.insert(points_faces_resized[0])
triangles = subdiv.getTriangleList().astype('int')

triangles_inds = []

for triangle in triangles:
    pt1 = (triangle[0], triangle[1])
    pt2 = (triangle[2], triangle[3])
    pt3 = (triangle[4], triangle[5])
    
    cv2.line(face, pt1, pt2, (0,0,255),1)
    cv2.line(face, pt2, pt3, (0,0,255),1)
    cv2.line(face, pt3, pt1, (0,0,255),1)
    
    triangles_inds.append([points_inds[pt1], points_inds[pt2], points_inds[pt3]])
    
plt.imshow(face)

<matplotlib.image.AxesImage at 0x121857438>

c. triangulation of the mean shape

mean_shape_points = [(int(i[0]), int(i[1]))for i in mean_shape_points]

points_inds_mean_shape = {i:mean_shape_points[i] for i in range(len(mean_shape_points))}

convexhull = cv2.convexHull(np.array(mean_shape_points))
mask_mean_shape = np.zeros_like(mean_shape_gray)
mask_mean_shape = cv2.cvtColor(mask_mean_shape, cv2.COLOR_GRAY2BGR)
mask_mean_shape = cv2.fillConvexPoly(mask_mean_shape, convexhull, (255,255,255))

for triangle_inds in triangles_inds:
    pt1 = points_inds_mean_shape[triangle_inds[0]]
    pt2 = points_inds_mean_shape[triangle_inds[1]]
    pt3 = points_inds_mean_shape[triangle_inds[2]]
    
    cv2.line(mask_mean_shape, pt1, pt2, (0,0,255),1)
    cv2.line(mask_mean_shape, pt2, pt3, (0,0,255),1)
    cv2.line(mask_mean_shape, pt3, pt1, (0,0,255),1)
    
plt.imshow(mask_mean_shape)

<matplotlib.image.AxesImage at 0x124d9de80>

step 5: swap face to the mean shape

plot_images([cv2.cvtColor(face,cv2.COLOR_BGR2GRAY), cv2.cvtColor(mask_mean_shape,cv2.COLOR_BGR2GRAY)], [points_faces_resized[0], mean_shape_points], rows=1, cols=2, size=(10,15))

a. convert white pixels in the mean shape to black pixels

for i in range(mask_mean_shape.shape[0]):
    for j in range(mask_mean_shape.shape[1]):
        if (mask_mean_shape[i][j] == [255,255,255]).all():
            mask_mean_shape[i][j] = np.array([0,0,0])
            
plt.imshow(mask_mean_shape)

<matplotlib.image.AxesImage at 0x124e05b00>

b. swap face patches to the mean shape

points_inds = {v:k for k,v in points_inds.items()}

for triangle_inds in triangles_inds:
    # calculate the new coordinates of points in each face patch
    points_face = np.array([points_inds[triangle_inds[0]],
                           points_inds[triangle_inds[1]],
                           points_inds[triangle_inds[2]]])
    
    points_mean_shape = np.array([points_inds_mean_shape[triangle_inds[0]],
                           points_inds_mean_shape[triangle_inds[1]],
                           points_inds_mean_shape[triangle_inds[2]]])
    
    # crop the face path in the original face
    convexhull = cv2.convexHull(points_face)
    rect = cv2.boundingRect(convexhull)
    (x, y, w, h) = rect
    
    cropped_triangle = face[y:y+h, x:x+w]
    cropped_triangle_mask = np.zeros((h,w), np.uint8)
    
    points_face_triangle = np.array([[points_face[0][0]-x, points_face[0][1]-y],
                               [points_face[1][0]-x, points_face[1][1]-y],
                               [points_face[2][0]-x, points_face[2][1]-y]])
    
    cropped_triangle_mask = cv2.fillConvexPoly(cropped_triangle_mask, points_face_triangle, 255)
    
    #plt.subplot(141)
    #plt.imshow(cropped_triangle_mask, cmap=plt.cm.gray)
    
    cropped_triangle = cv2.bitwise_and(cropped_triangle, cropped_triangle, mask=cropped_triangle_mask)
    
    #plt.subplot(142)
    #plt.imshow(cropped_triangle)
    
    # warp the face patch in the original face to the coresponding patch in the mean shape, using the Affine Transformation
    convexhull = cv2.convexHull(points_mean_shape)
    rect = cv2.boundingRect(convexhull)
    (x, y, w, h) = rect
    points_mean_shape_triangle = np.array([[points_mean_shape[0][0]-x, points_mean_shape[0][1]-y],
                               [points_mean_shape[1][0]-x, points_mean_shape[1][1]-y],
                               [points_mean_shape[2][0]-x, points_mean_shape[2][1]-y]])
    
    M = cv2.getAffineTransform(np.float32(points_face_triangle), np.float32(points_mean_shape_triangle))
    warped_face_triangle = cv2.warpAffine(cropped_triangle, M, (w,h))
    
    #plt.subplot(143)
    #plt.imshow(warped_face_triangle)
    
    # paste the warped face patch to the mean shape 
    triangle_rect_mean_shape = mask_mean_shape[y:y+h, x:x+w]
    triangle_rect_mean_shape = cv2.add(triangle_rect_mean_shape, warped_face_triangle)
    mask_mean_shape[y:y+h, x:x+w] = triangle_rect_mean_shape
    
    #plt.subplot(144)
    #plt.imshow(mask_mean_shape)
    
    #break

step 6: results

plt.subplot(121)
plt.imshow(face)
plt.title("the original face")

plt.subplot(122)
plt.imshow(mask_mean_shape)
plt.title("face swaped to the mean shape")

Text(0.5, 1.0, 'face swaped to the mean shape')

plt.imshow(cv2.subtract(face, mask_mean_shape))
plt.title("differences between the original face and the swaped face")

Text(0.5, 1.0, 'differences between the original face and the swaped face')