人脸识别2 FaceNet

1 Inception-ResNetV1网络

1.1 流程

input
 ↓
Stem
 ↓
Inception-resnet-A * 5
 ↓
Reduction-A
 ↓
Inception-resnet-B * 10
 ↓
Reduction-B
 ↓
Inception-resnet-C * 5
 ↓
AvgPooling
 ↓
Dropout(0.8)
 ↓
softmax

1.2 Stem

(1) Stem结构

input (160*160*3)
↓
Conv(32,3*3,2,v) (79*79*32)
↓
Conv(32,3*3,v) (77*77*32)
↓
Conv(64,3*3)  (77*77*64)
↓
MaxPool(s=2,v) (38*38*64)
↓
Conv(80,1*1) (38*38*80)
↓
Conv(192,3*3,v) (36*36*192)
↓
Conv(256,3*3,2,v) (17*17*256)

(2) Stem代码

inputs = Input(shape=input_shape)
# 160*160*3 -> 77,77,64
x = conv2d_bn(inputs,32,3,strides=2,padding='valid') # (160-3+1)/2=158/2=79
x = conv2d_bn(x,32,3,padding='valid') # 79-2=77
x = conv2d_bn(x,64,3)  # 77/1
# 77,77,64 -> 38,38,64
x = MaxPooling2D(3,strides=2)(x)  # (77-3)/2+1=38
# 38*38*64 -> 17,17,256
x = conv2d_bn(x,80,1,padding='valid')  # (38-1+1)/1 = 38
x = conv2d_bn(x,192,3,padding='valid') # (38-3+1)/1 = 36 
x = conv2d_bn(x,256,3,strides=2,padding='valid') # (36-3+1)/2 = 17

1.3 Inception-resnet-A

(1) Inception-resnet-A结构

input
↓
input->conv(32,1)->p1
input->conv(32,1)->conv(32,3)->p2
input->conv(32,1)->conv(32,3)->conv(32,3)->p3
↓
concatenate([p1,p2,p3])->conv(256,1) + input->Relu

(2) Inception-resnet-A代码

branch_0 = conv2d_bn(x, 32, 1)  
branch_1 = conv2d_bn(x, 32, 1)
branch_1 = conv2d_bn(branch_1, 32, 3)
branch_2 = conv2d_bn(x, 32, 1)
branch_2 = conv2d_bn(branch_2, 32, 3)
branch_2 = conv2d_bn(branch_2, 32, 3)
branches = [branch_0, branch_1, branch_2]

mixed = Concatenate(axis=channel_axis)(branches)
up = conv2d_bn(mixed,K.int_shape(x)[channel_axis],1,activation=None,use_bias=True)
up = Lambda(scaling,
         output_shape=K.int_shape(up)[1:],
         arguments={'scale': scale})(up)
x = add([x, up])
if activation is not None:
 x = Activation(activation)(x)

1.4 Reduction-A

(1) Reduction-A结构

input
↓
input->conv(384,3,s=2,v)->p1
input->conv(192,1)->conv(192,3)->conv(256,3,2,v)->p2
input->MaxPool(3,3,v)->p3
↓
concatenate([p1,p2,p3])

(2) Reduction-A代码

branch_0 = conv2d_bn(x, 384, 3,strides=2,padding='valid',name=name_fmt('Conv2d_1a_3x3', 0))
branch_1 = conv2d_bn(x, 192, 1, name=name_fmt('Conv2d_0a_1x1', 1))
branch_1 = conv2d_bn(branch_1, 192, 3, name=name_fmt('Conv2d_0b_3x3', 1))
branch_1 = conv2d_bn(branch_1,256,3,strides=2,padding='valid',name=name_fmt('Conv2d_1a_3x3', 1))
branch_pool = MaxPooling2D(3,strides=2,padding='valid',name=name_fmt('MaxPool_1a_3x3', 2))(x)
branches = [branch_0, branch_1, branch_pool]
x = Concatenate(axis=channel_axis, name='Mixed_6a')(branches)

1.5 Inception-resnet-B

(1) Inception-resnet-B结构

input
↓
input->conv(128,1)->p1
input->conv(128,1)->conv(128,1*7)->conv(128,7*1)->p2
↓
concatenate([p1,p2])->conv(896,1) + input->Relu

(2) Inception-resnet-B代码

branch_0 = conv2d_bn(x, 128, 1, name=name_fmt('Conv2d_1x1', 0))
branch_1 = conv2d_bn(x, 128, 1, name=name_fmt('Conv2d_0a_1x1', 1))
branch_1 = conv2d_bn(branch_1, 128, [1, 7], name=name_fmt('Conv2d_0b_1x7', 1))
branch_1 = conv2d_bn(branch_1, 128, [7, 1], name=name_fmt('Conv2d_0c_7x1', 1))
branches = [branch_0, branch_1]

mixed = Concatenate(axis=channel_axis, name=name_fmt('Concatenate'))(branches)
up = conv2d_bn(mixed,K.int_shape(x)[channel_axis],1,activation=None,use_bias=True,
             name=name_fmt('Conv2d_1x1'))
up = Lambda(scaling,
         output_shape=K.int_shape(up)[1:],
         arguments={'scale': scale})(up)
x = add([x, up])
if activation is not None:
 x = Activation(activation, name=name_fmt('Activation'))(x)

1. 6 Reduction-B

(1) Reduction-B结构

input
↓
input->conv(256,1)->conv(384,3,s=2,v)->p1
input->conv(256,1)->conv(256,3,2,v)->p2
input->conv(256,1)->conv(256,3)->conv(256,3,2,v)->p3
input->MaxPool(3,2,v)->p4
↓
concatenate([p1,p2,p3,p4])
 

(2) Reduction-B代码

name_fmt = partial(_generate_layer_name, prefix='Mixed_7a')
branch_0 = conv2d_bn(x, 256, 1, name=name_fmt('Conv2d_0a_1x1', 0))
branch_0 = conv2d_bn(branch_0,384,3,strides=2,padding='valid',name=name_fmt('Conv2d_1a_3x3', 0))
branch_1 = conv2d_bn(x, 256, 1, name=name_fmt('Conv2d_0a_1x1', 1))
branch_1 = conv2d_bn(branch_1,256,3,strides=2,padding='valid',name=name_fmt('Conv2d_1a_3x3', 1))
branch_2 = conv2d_bn(x, 256, 1, name=name_fmt('Conv2d_0a_1x1', 2))
branch_2 = conv2d_bn(branch_2, 256, 3, name=name_fmt('Conv2d_0b_3x3', 2))
branch_2 = conv2d_bn(branch_2,256,3,strides=2,padding='valid',name=name_fmt('Conv2d_1a_3x3', 2))
branch_pool = MaxPooling2D(3,strides=2,padding='valid',name=name_fmt('MaxPool_1a_3x3', 3))(x)
branches = [branch_0, branch_1, branch_2, branch_pool]
x = Concatenate(axis=channel_axis, name='Mixed_7a')(branches)

1.7 Inception-resnet-C

(1) Inception-resnet-C结构

input
↓
input->conv(128,1)->p1
input->conv(192,1)->conv(192,1*3)->conv(128,3*1)->p2
↓
concatenate([p1,p2])->conv(1792,1) + input->Relu
 

(2) Inception-resnet-C代码

branch_0 = conv2d_bn(x, 192, 1, name=name_fmt('Conv2d_1x1', 0))
branch_1 = conv2d_bn(x, 192, 1, name=name_fmt('Conv2d_0a_1x1', 1))
branch_1 = conv2d_bn(branch_1, 192, [1, 3], name=name_fmt('Conv2d_0b_1x3', 1))
branch_1 = conv2d_bn(branch_1, 192, [3, 1], name=name_fmt('Conv2d_0c_3x1', 1))
branches = [branch_0, branch_1]

mixed = Concatenate(axis=channel_axis, name=name_fmt('Concatenate'))(branches)
up = conv2d_bn(mixed,K.int_shape(x)[channel_axis],1,activation=None,use_bias=True,
             name=name_fmt('Conv2d_1x1'))
up = Lambda(scaling,
         output_shape=K.int_shape(up)[1:],
         arguments={'scale': scale})(up)
x = add([x, up])
if activation is not None:
 x = Activation(activation, name=name_fmt('Activation'))(x)

1.8 Inception-ResNetV1网络

(1) Inception-ResNetV1网络代码

from functools import partial
from keras.models import Model
from keras.layers import Activation
from keras.layers import BatchNormalization
from keras.layers import Concatenate
from keras.layers import Conv2D
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import GlobalAveragePooling2D
from keras.layers import Input
from keras.layers import Lambda
from keras.layers import MaxPooling2D
from keras.layers import add
from keras import backend as K


def scaling(x, scale):
 return x * scale

def _generate_layer_name(name, branch_idx=None, prefix=None):
 if prefix is None:
     return None
 if branch_idx is None:
     return '_'.join((prefix, name))
 return '_'.join((prefix, 'Branch', str(branch_idx), name))


def conv2d_bn(x,filters,kernel_size,strides=1,padding='same',activation='relu',use_bias=False,name=None):
 x = Conv2D(filters,
            kernel_size,
            strides=strides,
            padding=padding,
            use_bias=use_bias,
            name=name)(x)
 if not use_bias:
     x = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001,
                            scale=False, name=_generate_layer_name('BatchNorm', prefix=name))(x)
 if activation is not None:
     x = Activation(activation, name=_generate_layer_name('Activation', prefix=name))(x)
 return x


def _inception_resnet_block(x, scale, block_type, block_idx, activation='relu'):
 channel_axis = 3
 if block_idx is None:
     prefix = None
 else:
     prefix = '_'.join((block_type, str(block_idx)))
     
 name_fmt = partial(_generate_layer_name, prefix=prefix)

 if block_type == 'Block35':
     branch_0 = conv2d_bn(x, 32, 1, name=name_fmt('Conv2d_1x1', 0))
     branch_1 = conv2d_bn(x, 32, 1, name=name_fmt('Conv2d_0a_1x1', 1))
     branch_1 = conv2d_bn(branch_1, 32, 3, name=name_fmt('Conv2d_0b_3x3', 1))
     branch_2 = conv2d_bn(x, 32, 1, name=name_fmt('Conv2d_0a_1x1', 2))
     branch_2 = conv2d_bn(branch_2, 32, 3, name=name_fmt('Conv2d_0b_3x3', 2))
     branch_2 = conv2d_bn(branch_2, 32, 3, name=name_fmt('Conv2d_0c_3x3', 2))
     branches = [branch_0, branch_1, branch_2]
 elif block_type == 'Block17':
     branch_0 = conv2d_bn(x, 128, 1, name=name_fmt('Conv2d_1x1', 0))
     branch_1 = conv2d_bn(x, 128, 1, name=name_fmt('Conv2d_0a_1x1', 1))
     branch_1 = conv2d_bn(branch_1, 128, [1, 7], name=name_fmt('Conv2d_0b_1x7', 1))
     branch_1 = conv2d_bn(branch_1, 128, [7, 1], name=name_fmt('Conv2d_0c_7x1', 1))
     branches = [branch_0, branch_1]
 elif block_type == 'Block8':
     branch_0 = conv2d_bn(x, 192, 1, name=name_fmt('Conv2d_1x1', 0))
     branch_1 = conv2d_bn(x, 192, 1, name=name_fmt('Conv2d_0a_1x1', 1))
     branch_1 = conv2d_bn(branch_1, 192, [1, 3], name=name_fmt('Conv2d_0b_1x3', 1))
     branch_1 = conv2d_bn(branch_1, 192, [3, 1], name=name_fmt('Conv2d_0c_3x1', 1))
     branches = [branch_0, branch_1]

 mixed = Concatenate(axis=channel_axis, name=name_fmt('Concatenate'))(branches)
 up = conv2d_bn(mixed,K.int_shape(x)[channel_axis],1,activation=None,use_bias=True,
                name=name_fmt('Conv2d_1x1'))
 up = Lambda(scaling,
             output_shape=K.int_shape(up)[1:],
             arguments={'scale': scale})(up)
 x = add([x, up])
 if activation is not None:
     x = Activation(activation, name=name_fmt('Activation'))(x)
 return x


def InceptionResNetV1(input_shape=(160, 160, 3),
                   classes=128,
                   dropout_keep_prob=0.8):
 channel_axis = 3
 inputs = Input(shape=input_shape)
 # 160,160,3 -> 77,77,64
 x = conv2d_bn(inputs, 32, 3, strides=2, padding='valid', name='Conv2d_1a_3x3')
 x = conv2d_bn(x, 32, 3, padding='valid', name='Conv2d_2a_3x3')
 x = conv2d_bn(x, 64, 3, name='Conv2d_2b_3x3')
 # 77,77,64 -> 38,38,64
 x = MaxPooling2D(3, strides=2, name='MaxPool_3a_3x3')(x)

 # 38,38,64 -> 17,17,256
 x = conv2d_bn(x, 80, 1, padding='valid', name='Conv2d_3b_1x1')
 x = conv2d_bn(x, 192, 3, padding='valid', name='Conv2d_4a_3x3')
 x = conv2d_bn(x, 256, 3, strides=2, padding='valid', name='Conv2d_4b_3x3')

 # 5x Block35 (Inception-ResNet-A block):
 for block_idx in range(1, 6):
     x = _inception_resnet_block(x,scale=0.17,block_type='Block35',block_idx=block_idx)

 # Reduction-A block:
 # 17,17,256 -> 8,8,896
 name_fmt = partial(_generate_layer_name, prefix='Mixed_6a')
 branch_0 = conv2d_bn(x, 384, 3,strides=2,padding='valid',name=name_fmt('Conv2d_1a_3x3', 0))
 branch_1 = conv2d_bn(x, 192, 1, name=name_fmt('Conv2d_0a_1x1', 1))
 branch_1 = conv2d_bn(branch_1, 192, 3, name=name_fmt('Conv2d_0b_3x3', 1))
 branch_1 = conv2d_bn(branch_1,256,3,strides=2,padding='valid',name=name_fmt('Conv2d_1a_3x3', 1))
 branch_pool = MaxPooling2D(3,strides=2,padding='valid',name=name_fmt('MaxPool_1a_3x3', 2))(x)
 branches = [branch_0, branch_1, branch_pool]
 x = Concatenate(axis=channel_axis, name='Mixed_6a')(branches)

 # 10x Block17 (Inception-ResNet-B block):
 for block_idx in range(1, 11):
     x = _inception_resnet_block(x,
                                 scale=0.1,
                                 block_type='Block17',
                                 block_idx=block_idx)

 # Reduction-B block
 # 8,8,896 -> 3,3,1792
 name_fmt = partial(_generate_layer_name, prefix='Mixed_7a')
 branch_0 = conv2d_bn(x, 256, 1, name=name_fmt('Conv2d_0a_1x1', 0))
 branch_0 = conv2d_bn(branch_0,384,3,strides=2,padding='valid',name=name_fmt('Conv2d_1a_3x3', 0))
 branch_1 = conv2d_bn(x, 256, 1, name=name_fmt('Conv2d_0a_1x1', 1))
 branch_1 = conv2d_bn(branch_1,256,3,strides=2,padding='valid',name=name_fmt('Conv2d_1a_3x3', 1))
 branch_2 = conv2d_bn(x, 256, 1, name=name_fmt('Conv2d_0a_1x1', 2))
 branch_2 = conv2d_bn(branch_2, 256, 3, name=name_fmt('Conv2d_0b_3x3', 2))
 branch_2 = conv2d_bn(branch_2,256,3,strides=2,padding='valid',name=name_fmt('Conv2d_1a_3x3', 2))
 branch_pool = MaxPooling2D(3,strides=2,padding='valid',name=name_fmt('MaxPool_1a_3x3', 3))(x)
 branches = [branch_0, branch_1, branch_2, branch_pool]
 x = Concatenate(axis=channel_axis, name='Mixed_7a')(branches)

 # 5x Block8 (Inception-ResNet-C block):
 for block_idx in range(1, 6):
     x = _inception_resnet_block(x,
                                 scale=0.2,
                                 block_type='Block8',
                                 block_idx=block_idx)
 x = _inception_resnet_block(x,scale=1.,activation=None,block_type='Block8',block_idx=6)

 # 平均池化
 x = GlobalAveragePooling2D(name='AvgPool')(x)
 x = Dropout(1.0 - dropout_keep_prob, name='Dropout')(x)
 # 全连接层到128
 x = Dense(classes, use_bias=False, name='Bottleneck')(x)
 bn_name = _generate_layer_name('BatchNorm', prefix='Bottleneck')
 x = BatchNormalization(momentum=0.995, epsilon=0.001, scale=False,
                        name=bn_name)(x)

 # 创建模型
 model = Model(inputs, x, name='inception_resnet_v1')

 return model

2 人脸识别之FaceNet

人脸识别流程：提取人脸->提取人脸特征向量 -> L2标准化得到人脸特征-> 人脸识别

1. 提取人脸
2. 提取人脸特征向量
3. L2标准化得到人脸特征
4. 人脸识别

import numpy as np
import cv2
from net.inception import InceptionResNetV1
# from keras.models import load_model
# import face_recognition
#---------------------------------#
#   图片预处理
#   高斯归一化
#---------------------------------#
def pre_process(x):
    if x.ndim == 4:
        axis = (1, 2, 3)
        size = x[0].size
    elif x.ndim == 3:
        axis = (0, 1, 2)
        size = x.size
    else:
        raise ValueError('Dimension should be 3 or 4')

    mean = np.mean(x, axis=axis, keepdims=True)
    std = np.std(x, axis=axis, keepdims=True)
    std_adj = np.maximum(std, 1.0/np.sqrt(size))
    y = (x - mean) / std_adj
    return y
#---------------------------------#
#   l2标准化
#---------------------------------#
def l2_normalize(x, axis=-1, epsilon=1e-10):
    output = x / np.sqrt(np.maximum(np.sum(np.square(x), axis=axis, keepdims=True), epsilon))
    return output
#---------------------------------#
#   计算128特征值
#---------------------------------#
def calc_128_vec(model,img):
    face_img = pre_process(img)
    pre = model.predict(face_img)
    pre = l2_normalize(np.concatenate(pre))
    pre = np.reshape(pre,[1,128])
    return pre
#---------------------------------#
#   获取人脸框
#---------------------------------#
def get_face_img(cascade,filepaths,margin):

    aligned_images = []

    img = cv2.imread(filepaths)
    img = cv2.cvtColor(img,cv2.COLOR_BGRA2RGB)

    faces = cascade.detectMultiScale(img,
                                        scaleFactor=1.1,
                                        minNeighbors=3)
    (x, y, w, h) = faces[0]
    # print(x, y, w, h)
    cropped = img[y-margin//2:y+h+margin//2,
                    x-margin//2:x+w+margin//2, :]
    aligned = cv2.resize(cropped, (160, 160))
    aligned_images.append(aligned)
        
    return np.array(aligned_images)
#---------------------------------#
#   计算人脸距离
#---------------------------------#
def face_distance(face_encodings, face_to_compare):
    if len(face_encodings) == 0:
        return np.empty((0))

    return np.linalg.norm(face_encodings - face_to_compare, axis=1)

if __name__ == "__main__":
    cascade_path = './model/haarcascade_frontalface_alt2.xml'
    cascade = cv2.CascadeClassifier(cascade_path)
    
    image_size = 160
    model = InceptionResNetV1()
    # model.summary()
    model_path = './model/facenet_keras.h5'
    model.load_weights(model_path)
    
    img1 = get_face_img(cascade,r"img/0.jpg",10)
    img2 = get_face_img(cascade,r"img/1.jpg",10)
    img3 = get_face_img(cascade,r"img/2.jpg",10)
    
    print(face_distance(calc_128_vec(model,img1),calc_128_vec(model,img2)))
    print(face_distance(calc_128_vec(model,img2),calc_128_vec(model,img3)))


# [1.1135937]
# [0.72458065]