用matlab搭建vgg,基于VGG-face网络结构的特征提取和人脸识别-作业2

参考文献：

Deep face recognition, O. M. Parkhi and A. Vedaldi and A. Zisserman,

Proceedings of the British Machine Vision Conference (BMVC), 2015 (paper).

利用vgg-face网络结构，去掉了最后一层全连接，提取人脸特征，实现人脸识别及landmark

网络权重下载：http://www.robots.ox.ac.uk/~vgg/software/vgg_face/

代码实现：

# -*- coding: utf-8 -*-

import cv2

import dlib

import numpy as np

import matplotlib.pyplot as plt

from PIL import Image

import math

imgpath='./classifer/posimage/2.jpg'

bgr_img=cv2.imread(imgpath,1)

rgb_img=cv2.cvtColor(bgr_img, cv2.COLOR_BGR2RGB)

#gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

'''implement the face landmark estimation and pose alignment'''

#face landmark estimation

detector = dlib.get_frontal_face_detector()

landmark_predictor = dlib.shape_predictor('shape_predictor_68_face_landmarks.dat')

dets = detector(rgb_img,1)

##show the result of face landmark

#if (len(dets) > 0):

# for k,d in enumerate(dets):

# cv2.rectangle(bgr_img,(d.left(),d.top()),(d.right(),d.bottom()),(255,255,255))

# shape = landmark_predictor(bgr_img,d)

# for i in range(68):

# cv2.circle(bgr_img, (shape.part(i).x, shape.part(i).y),1,(0,255,0), -1, 8)

# cv2.putText(bgr_img,str(i),(shape.part(i).x,shape.part(i).y),cv2.FONT_HERSHEY_SIMPLEX,0.2,(255,2555,255))

#cv2.imshow('Frame',bgr_img)

#cv2.waitKey(0)

faces=dlib.full_object_detections()

for det in dets:

faces.append(landmark_predictor(rgb_img,det))

#pose alignment

images=dlib.get_face_chips(rgb_img,faces,size=320)

image_cnt=0

for image in images:

image_cnt += 1

cv_rgb_image = np.array(image).astype(np.uint8)# 先转换为numpy数组

cv_bgr_image = cv2.cvtColor(cv_rgb_image, cv2.COLOR_RGB2BGR)# opencv下颜色空间为bgr，所以从rgb转换为bgr

cv2.imshow('%s'%(image_cnt), cv_bgr_image)

cv2.waitKey(0)

'''

use VGG-face model from

Deep face recognition, O. M. Parkhi and A. Vedaldi and A. Zisserman,

Proceedings of the British Machine Vision Conference (BMVC), 2015 (paper).

'''

from keras.models import Sequential, Model

from keras.layers import Flatten, Dropout, Activation, Permute

from keras.layers import Conv2D, MaxPooling2D

from keras import backend as K

K.set_image_data_format( 'channels_last' )

def convblock(cdim, nb, bits=3):

L = []

for k in range(1,bits+1):

convname = 'conv'+str(nb)+'_'+str(k)

L.append( Conv2D(cdim,(3,3),padding='same',activation='relu',name=convname) )

L.append( MaxPooling2D((2,2), strides=(2,2)) )

return L

def vgg_face_blank():

withDO = True

if True:

mdl = Sequential()

mdl.add( Permute((1,2,3), input_shape=(224,224,3)) )

for l in convblock(64,1, bits=2):

mdl.add(l)

for l in convblock(128,2, bits=2):

mdl.add(l)

for l in convblock(256,3, bits=3):

mdl.add(l)

for l in convblock(512,4, bits=3):

mdl.add(l)

for l in convblock(512,5, bits=3):

mdl.add(l)

mdl.add( Conv2D(4096,kernel_size=(7,7),activation='relu',name='fc6'))

if withDO:

mdl.add(Dropout(0.5))

mdl.add( Conv2D(4096,kernel_size=(1,1),activation='relu',name='fc7'))

if withDO:

mdl.add(Dropout(0.5))

mdl.add(Conv2D(2622,kernel_size=(1,1),activation='relu',name='fc8'))

mdl.add(Flatten())

mdl.add(Activation('softmax'))

return mdl

else:

raise ValueError('not implemented')

facemodel = vgg_face_blank()

facemodel.summary()

from scipy.io import loadmat

data = loadmat('vgg-face.mat', matlab_compatible=False, struct_as_record=False)

l = data['layers']

description = data['meta'][0,0].classes[0,0].description

def copy_mat_to_keras(kmodel):

kerasnames = [lr.name for lr in kmodel.layers]

prmt = (0,1,2,3)

for i in range(l.shape[1]):

matname = l[0,i][0,0].name[0]

if matname in kerasnames:

kindex = kerasnames.index(matname)

l_weights = l[0,i][0,0].weights[0,0]

l_bias = l[0,i][0,0].weights[0,1]

f_l_weights = l_weights.transpose(prmt)

assert (f_l_weights.shape == kmodel.layers[kindex].get_weights()[0].shape)

assert (l_bias.shape[1] == 1)

assert (l_bias[:,0].shape == kmodel.layers[kindex].get_weights()[1].shape)

assert (len(kmodel.layers[kindex].get_weights()) == 2)

kmodel.layers[kindex].set_weights([f_l_weights, l_bias[:,0]])

copy_mat_to_keras(facemodel)

#from keras.models import load_model

#facemodel.save('model_weight.h5')

im = Image.open('test.jpg')

im = im.resize((224,224))

plt.imshow(np.asarray(im))

def pred(kmodel, crpimg):

imarr = np.array(crpimg).astype(np.float32)

imarr = np.expand_dims(imarr, axis=0)

out = kmodel.predict(imarr)

best_index = np.argmax(out, axis=1)[0]

best_name = description[best_index,0]

print(best_index, best_name[0], out[0,best_index], [np.min(out), np.max(out)])

crpim = im

pred(facemodel, crpim)

#face feature:drop the last layer

featuremodel = Model(inputs=facemodel.layers[0].input, outputs=facemodel.layers[-2].output)

featuremodel.summary()

def features(featmodel, crpimg):

imarr = np.array(crpimg).astype(np.float32)

imarr = np.expand_dims(imarr, axis=0)

fvec = featmodel.predict(imarr)[0,:]

normfvec = math.sqrt(fvec.dot(fvec))

return fvec/normfvec

crpimg=Image.open('./classifer/bounding box/posimage/3.jpg')

crpimg=crpimg.resize((224,224))

sample_feature=features(featuremodel, crpimg)

print(sample_feature)

'''train classifier from the embedded network features/measurements'''

'''without landmark estimation and pose alignment'''

import os

pos_im_file="./classifer/bounding box/posimage"

neg_im_file="./classifer/bounding box/negimage"

'''

if want to compare with and without the pose alignment

just modify the path

'''

#pos_im_file="./classifer/bounding box with only fl/posimage"

#neg_im_file="./classifer/bounding box with only fl/negimage"

#pos_im_file="./classifer/bounding box with fl and pa/posimage"

#neg_im_file="./classifer/bounding box with fl and pa/negimage"

def loadimg_infile(imgdir):

imgname=[]

imgname_sorted=[]

filename=os.listdir(imgdir)

for file in filename:

if 'jpg' in file:

imgname.append(imgdir+file)

s=len(imgname)

for i in range(s):

imgname_sorted.append(imgdir+'/'+str(i+1)+'.jpg')

return imgname_sorted

pos_im_list=loadimg_infile(pos_im_file)

neg_im_list=loadimg_infile(neg_im_file)

xdata=[]

ydata=[]

for pos_im in pos_im_list:

crpimg=Image.open(pos_im)

crpimg=crpimg.resize((224,224))

xdata.append( features(featuremodel, crpimg))

ydata.append('1.0') #is Obama

for neg_im in neg_im_list:

crpimg=Image.open(neg_im)

crpimg=crpimg.resize((224,224))

xdata.append( features(featuremodel, crpimg))

ydata.append('0.0') #not Obama

'''split data to train and test'''

from sklearn.cross_validation import train_test_split

test_size=0.1

X_train,X_test,y_train,y_test=train_test_split(xdata,ydata,test_size=test_size,random_state=1)

'''different kinds of classifers'''

import matplotlib.pyplot as plt

from sklearn.naive_bayes import GaussianNB

from sklearn.naive_bayes import MultinomialNB

from sklearn.linear_model import LogisticRegression

from sklearn.neighbors import KNeighborsClassifier

from sklearn.svm import SVC

from sklearn import metrics

from sklearn.metrics import precision_recall_curve, roc_curve, auc

from sklearn.externals import joblib

#clf = GaussianNB()

#clf = MultinomialNB(alpha=0.01)

#clf = LogisticRegression()

#clf = SVC(kernel = 'linear')

#clf = KNeighborsClassifier()

clf = SVC(kernel='linear', gamma= 0.5, C = 2.0)

clf.fit(X_train,y_train)

#clf = joblib.load("week4_model.m")

pred_train = clf.predict(X_train)

pred_test = clf.predict(X_test)

s_train= metrics.confusion_matrix(y_train,pred_train,labels=None)

s_test= metrics.confusion_matrix(y_test,pred_test,labels=None)

accuracy_train=(s_train[0][0]+s_train[1][1])/(100-100*test_size)

accuracy_test=(s_test[0][0]+s_test[1][1])/(100*test_size)

print('accuracy_train:',accuracy_train)

print('accuracy_test:',accuracy_test)

#joblib.dump(clf, "week4_model.m")

效果：

2.jpg

3.jpg

4.jpg

5.jpg