使用TensorFlow2.0中Keras进行迁移学习识别验证码

网上有很多文章介绍过迁移学习，但TensorFlow2.0的较少，有TensorFlow2.0的，但能说清楚明白的更少。验证码的样本需要自行标记，本文使用的样本：训练集1500，验证集500，测试集500。本次迁移学习选择的预训练模型为：Xception，是个比较优秀的CNN模型。验证码的特征是带空心字符、有大小写且粘连的验证码。环境使用的是MX350 2G的GPU（使用GPU训练很快）。

主要步骤分为四步：

1）将Xception预训练模型作为特征抽取器，即去掉头部的输出层（默认是1000分类），增加平均池化层、Dropout层和4个并行（为何不直接36**4类输出呢？主要是为了减少参数，减少计算量）的多分类输出层，训练出一个微调模型，使用的是imagenet的权重weights（这就是微调的价值，不需要大样本，不需要大量计算）；

import numpy as np
import glob
from keras.applications.xception import Xception,preprocess_input
from keras.layers import Input,Dense,Dropout
from keras.models import Model
from scipy import misc
from PIL import Image
import imageio
from tensorflow import keras

train_samples = glob.glob('E:/jcaptcha/train/*.jpg')
valid_samples = glob.glob('E:/jcaptcha/valid/*.jpg')
letter_list = [chr(i) for i in range(48, 58)] + [chr(i) for i in range(65, 91)]   # 需要识别的36类
img_size = (150, 150)
input_image = Input(shape=(img_size[0],img_size[1],3))

base_model = Xception(input_tensor=input_image, weights="imagenet", include_top=False, pooling='avg')
base_model.trainable = False
predicts = [Dense(36, activation='softmax')(Dropout(0.5)(base_model.output)) for i in range(4)]

model = Model(inputs=input_image, outputs=predicts)
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])

def data_generator(data, batch_size): #样本生成器，节省内存
    while True:
        #np.random.choice(x,y)生成一个从x中抽取的随机数,维度为y的向量，y为抽取次数
        batch = np.random.choice(data, batch_size)
        x,y = [],[]
        for img in batch:
            x.append(np.array(Image.fromarray(imageio.imread(img)).resize(img_size)))  # 读取resize图片,再存进x列表
            real_num = img[-8:-4].upper() #标签,如：L962
            y_list = []
            for i in real_num:
                if ord(i) - ord('A') >= 0:
                    y_list.append(ord(i) - ord('A') + 10)
                else:
                    y_list.append(ord(i) - ord('0'))
            #print(real_num,y_list) L962 [21, 9, 6, 2]
            y.append(y_list)
 
        #把验证码标签添加到y列表,ord(i)-ord('A')把对应字母转化为数字a=10，b=11……z=35
        x = preprocess_input(np.array(x).astype(float))
        #原先是dtype=uint8转成一个纯数字的array
        y = np.array(y)
        yield x,[y[:,i] for i in range(4)]
        
model.fit(data_generator(train_samples, 10), steps_per_epoch=1000, epochs=5, validation_data=data_generator(valid_samples, 10), validation_steps=200)

#保存模型
model.save('E:/Fine_Tune_Model.h5')

2）对微调模型进行训练，仍然用这些数据进行训练（因为样本比较少，也可以采用新的样本），使用的是微调模型的权重weights，训练出一个更好的微调模型；

import numpy as np
from scipy import misc
from keras.applications.xception import Xception,preprocess_input
import glob
from keras.layers import Input,Dense,Dropout
from keras.models import Model
from PIL import Image
import imageio

img_size = (150, 150)
input_image = Input(shape=(img_size[0],img_size[1],3))
base_model = Xception(input_tensor=input_image, weights=None, include_top=False, pooling='avg')
predicts = [Dense(36, activation='softmax')(Dropout(0.5)(base_model.output)) for i in range(4)]
model = Model(inputs=input_image, outputs=predicts)
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
model.load_weights('E:/Fine_Tune_Model8.h5')
def data_generator(data, batch_size): #样本生成器，节省内存
    while True:
        #np.random.choice(x,y)生成一个从x中抽取的随机数,维度为y的向量，y为抽取次数
        batch = np.random.choice(data, batch_size)
        x,y = [],[]
        for img in batch:
            x.append(np.array(Image.fromarray(imageio.imread(img)).resize(img_size)))  # 读取resize图片,再存进x列表
            real_num = img[-8:-4].upper()
            y_list = []
            for i in real_num:
                if ord(i) - ord('A') >= 0:
                    y_list.append(ord(i) - ord('A') + 10)
                else:
                    y_list.append(ord(i) - ord('0'))
            #print(real_num,y_list) L962 [21, 9, 6, 2]
            y.append(y_list)
 
        x = preprocess_input(np.array(x).astype(float))
        #原先是dtype=uint8转成一个纯数字的array
        y = np.array(y)
        yield x,[y[:,i] for i in range(4)]
        #输出：图片array和四个转化成数字的字母 例如：[array([6]), array([0]), array([3]), array([24])])
#获取指定目录下的所有图片
train_samples = glob.glob('E:/jcaptcha/train/*.jpg')
valid_samples = glob.glob('E:/jcaptcha/valid/*.jpg')

#Continue training
model.fit(data_generator(train_samples, 10), steps_per_epoch=100, epochs=10, validation_data=data_generator(valid_samples, 10), validation_steps=10)
model.save('E:/Fine_Tune_Model9.h5')

3）反复进行2）直至得到一个比较满意的微调模型N（4个分类的精准率accuracy相乘接近1.0）;

4）使用微调模型N通过测试集进行测试验证微调模型N的效果。

from keras.models import load_model
import numpy as np
from scipy import misc
from keras.applications.xception import preprocess_input
import matplotlib.pyplot as plt # plt 用于显示图片
import matplotlib.image as mpimg # mpimg 用于读取图片
import glob
from PIL import Image
import imageio

img_size = (150, 150)
model = load_model('E:/Fine_Tune_Model9.h5')

letter_list = [chr(i) for i in range(48,58)] + [chr(i) for i in range(65,91)]
print(letter_list)

def data_generator_test(data, n): #样本生成器，节省内存
    while True:
        batch = np.array([data[n]])
        x,y = [],[]
        for img in batch:
            x.append(np.array(Image.fromarray(imageio.imread(img)).resize((150,150))))
            y_list = []
            real_num = img[-8:-4].upper()
            for i in real_num:
                if ord(i)-ord('A') >= 0:
                    y_list.append(ord(i)-ord('A')+10)
                else:
                    y_list.append(ord(i)-ord('0'))
            #print('real_1:',img[-8:-4])
        x = preprocess_input(np.array(x).astype(float)) #原先是dtype=uint8转成一个纯数字的array
        y = np.array(y)
        yield x,[y[:,i] for i in range(4)]
def decode(y):
    y = np.argmax(np.array(y), axis=2)[:,0]
    print(y)
    return ''.join([letter_list[x] for x in y])

def predict2(n):
    x,y = next(data_generator_test(test_samples, n))
    pred = model.predict(x)
    v=[]
    r=[]
    for i in range(4):
        if y[i][0]>=10:
            r.append(chr(ord('A')+y[i][0]-10))
        else:
            r.append(chr(ord('0')+y[i][0]))
    #输出测试结果
    real = ''
    for i in r:
        real += i
    return (decode(pred),real)
test_samples = glob.glob('E:/jcaptcha/test/*.jpg')
print(len(test_samples))
n = 0
n_right = 0
for i in range(10):
    n += 1
    print('========第%d个验证码======='%(n))
    predict,real = predict2(i)
    if real == predict:
        n_right += 1
        print("预测成功")
    else:
        print("预测失败")
    print('real:', real)
    print('predict:',predict)
    image = mpimg.imread(test_samples[i])
    plt.axis('off')
    plt.imshow(image)
    plt.show()
    print('========第%d个验证码======='%(n))
print('总数',n,'成功数',n_right,'成功率',n_right*100/n,"%")

经过8次反复训练，在测试集上识别的成功率可以达到80%。