mnist各种网络研究1 实验记录

这次主要看看修改网络结构后，acc、参数个数、运算量之间的关系。
模型均用keras编写，sgd或者adam优化，4显卡并行训练，batch_size=2000（性能会降低，但是训练速度快）
下面先列举训练的各种网络
Net1：直接softmax分为10类
test accuracy 0.9169

Net2：2层全连接层
test accuracy 0.9843

Net3：标准卷积网络（本网络训练adam比sgd结果要好）
test accuracy 0.9921

Net4 标准网络去掉maxpool，采用stride=(2,2)方式
test accuracy 0.9881

Net5 使用可分离卷积
test accuracy 0.9877

with tf.device('/cpu:0'):
    input_image = Input(shape=(28,28,1))
    cnn = SeparableConv2D(32, (3,3),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn = SeparableConv2D(64, (3,3),strides=(2,2),padding='same',activation='relu')(cnn)
    cnn = Flatten()(cnn)
    feature = Dense(1024,activation='relu')(cnn)
    feature = Dropout(0.5)(feature)
    predict = Dense(num_classes,activation='softmax',name='softmax')(feature) 
    model = Model(inputs=input_image, outputs=predict)
adam = Adam(lr=1e-4)
model.compile(optimizer=adam,loss='sparse_categorical_crossentropy',metrics=['accuracy'])

Net6 Net5，使用可分离卷积卷积到1024
test accuracy 0.9715

with tf.device('/cpu:0'):
    input_image = Input(shape=(28,28,1))
    cnn = SeparableConv2D(32, (3,3),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn = SeparableConv2D(64, (3,3),strides=(2,2),padding='same',activation='relu')(cnn)
    cnn = SeparableConv2D(1024, (7,7),padding='valid',activation='relu')(cnn)    
    cnn = Flatten()(cnn)
    feature = Dropout(0.5)(cnn)
    predict = Dense(num_classes,activation='softmax',name='softmax')(feature) 
    model = Model(inputs=input_image, outputs=predict)

adam = Adam(lr=1e-4)
model.compile(optimizer=adam,loss='sparse_categorical_crossentropy',metrics=['accuracy'])

Net7 Net6使用改变学习率的sgd
test accuracy 0.9748

sgd = optimizers.SGD(lr=.1, momentum=0.9, nesterov=True)
model.compile(optimizer=sgd,loss='sparse_categorical_crossentropy',metrics=['accuracy'])
model_p = multi_gpu_model(model,4)
model_p.compile(optimizer=sgd,loss='sparse_categorical_crossentropy',metrics=['accuracy'])

def scheduler_200_81_122(epoch):
    if epoch < 121:
        return 0.01
    if epoch < 162:
        return 0.001
    return 0.0001

cb_lr = LearningRateScheduler(scheduler_200_81_122)
cbks = [cb_lr]

model_p.fit(x_train,y_train,batch_size=2000,epochs=200,
    callbacks=cbks,
    validation_data=(x_test, y_test))

Net8 Net7修改卷积核5*5
test accuracy 0.9788

with tf.device('/cpu:0'):
    input_image = Input(shape=(28,28,1))
    cnn = SeparableConv2D(32, (5,5),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn = SeparableConv2D(64, (5,5),strides=(2,2),padding='same',activation='relu')(cnn)
    cnn = SeparableConv2D(1024, (7,7),padding='valid',activation='relu')(cnn)    
    cnn = Flatten()(cnn)
    feature = Dropout(0.5)(cnn)
    predict = Dense(num_classes,activation='softmax',name='softmax')(feature) 
    model = Model(inputs=input_image, outputs=predict)

Net9 Net8通道全部减小到50%
test accuracy 0.9735

with tf.device('/cpu:0'):
    input_image = Input(shape=(28,28,1))
    cnn = SeparableConv2D(16, (5,5),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn = SeparableConv2D(32, (5,5),strides=(2,2),padding='same',activation='relu')(cnn)
    cnn = SeparableConv2D(512, (7,7),padding='valid',activation='relu')(cnn)    
    cnn = Flatten()(cnn)
    feature = Dropout(0.5)(cnn)
    predict = Dense(num_classes,activation='softmax',name='softmax')(feature) 
    model = Model(inputs=input_image, outputs=predict)

Net10 Net9通道全部减小到50%
test accuracy 0.9687

with tf.device('/cpu:0'):
    input_image = Input(shape=(28,28,1))
    cnn = SeparableConv2D(8, (5,5),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn = SeparableConv2D(16, (5,5),strides=(2,2),padding='same',activation='relu')(cnn)
    cnn = SeparableConv2D(256, (7,7),padding='valid',activation='relu')(cnn)    
    cnn = Flatten()(cnn)
    feature = Dropout(0.5)(cnn)
    predict = Dense(num_classes,activation='softmax',name='softmax')(feature) 
    model = Model(inputs=input_image, outputs=predict)

Net11 2个Net10合并
test accuracy 0.9716

with tf.device('/cpu:0'):
    input_image = Input(shape=(28,28,1))
    
    cnn = SeparableConv2D(8, (5,5),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn = SeparableConv2D(16, (5,5),strides=(2,2),padding='same',activation='relu')(cnn)
    cnn = SeparableConv2D(256, (7,7),padding='valid',activation='relu')(cnn)  
    cnn = Flatten()(cnn)
    
    cnn2 = SeparableConv2D(8, (5,5),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn2 = SeparableConv2D(16, (5,5),strides=(2,2),padding='same',activation='relu')(cnn2)
    cnn2 = SeparableConv2D(256, (7,7),padding='valid',activation='relu')(cnn2)
    cnn2 = Flatten()(cnn2)
    
    cnn = concatenate([cnn,cnn2],axis=1)
    feature = Dropout(0.5)(cnn)
    predict = Dense(num_classes,activation='softmax',name='softmax')(cnn) 
    model = Model(inputs=input_image, outputs=predict)

Net12 Net10和Net1修改版合并
test accuracy 0.9675

with tf.device('/cpu:0'):
    input_image = Input(shape=(28,28,1))
    
    cnn = SeparableConv2D(8, (5,5),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn = SeparableConv2D(16, (5,5),strides=(2,2),padding='same',activation='relu')(cnn)
    cnn = SeparableConv2D(256, (7,7),padding='valid',activation='relu')(cnn)  
    cnn = Flatten()(cnn)
    
    F2 = Flatten()(input_image)
    F2 = Dense(256,activation='relu')(F2)
    
    cnn = concatenate([cnn,F2],axis=1)
    feature = Dropout(0.5)(cnn)
    predict = Dense(num_classes,activation='softmax',name='softmax')(cnn) 
    model = Model(inputs=input_image, outputs=predict)

Net13 2个不同参数的Net10合并
test accuracy 0.9661

with tf.device('/cpu:0'):
    input_image = Input(shape=(28,28,1))
    
    cnn = SeparableConv2D(8, (5,5),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn = SeparableConv2D(16, (5,5),strides=(2,2),padding='same',activation='relu')(cnn)
    cnn = SeparableConv2D(256, (7,7),padding='valid',activation='relu')(cnn)  
    cnn = Flatten()(cnn)
    
    cnn2 = SeparableConv2D(8, (3,3),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn2 = SeparableConv2D(16, (3,3),strides=(2,2),padding='same',activation='relu')(cnn2)
    cnn2 = SeparableConv2D(256, (7,7),padding='valid',activation='relu')(cnn2)
    cnn2 = Flatten()(cnn2)
    
    cnn = concatenate([cnn,cnn2],axis=1)
    feature = Dropout(0.5)(cnn)
    predict = Dense(num_classes,activation='softmax',name='softmax')(cnn) 
    model = Model(inputs=input_image, outputs=predict)

Net14 Net10修改为77卷积
test accuracy 0.9723

Net15 Net14上修改第二层卷积为11 33 55合并
test accuracy 0.9759

with tf.device('/cpu:0'):
    input_image = Input(shape=(28,28,1))
    cnn = SeparableConv2D(8, (7,7),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn1 = Conv2D(16, (1,1),strides=(2,2),padding='same',activation='relu')(cnn)
    cnn2 = SeparableConv2D(16, (5,5),strides=(2,2),padding='same',activation='relu')(cnn)
    cnn3 = SeparableConv2D(16, (3,3),strides=(2,2),padding='same',activation='relu')(cnn)

    cnn = concatenate([cnn1,cnn2,cnn3],axis=3)
    cnn = SeparableConv2D(256, (7,7),padding='valid',activation='relu')(cnn)
    cnn = Flatten()(cnn)

    feature = Dropout(0.5)(cnn)
    predict = Dense(num_classes,activation='softmax',name='softmax')(feature) 
    model = Model(inputs=input_image, outputs=predict)

Net16 Net14上修改最后为标准卷积方式
test accuracy 0.982

with tf.device('/cpu:0'):
    input_image = Input(shape=(28,28,1))
    
    cnn = SeparableConv2D(8, (7,7),strides=(2,2),padding='same',activation='relu')(input_image)
    cnn = SeparableConv2D(16, (7,7),strides=(2,2),padding='same',activation='relu')(cnn)
    cnn = Conv2D(256, (7,7),padding='valid',activation='relu')(cnn)  
    cnn = Flatten()(cnn)

    feature = Dropout(0.5)(cnn)
    predict = Dense(num_classes,activation='softmax',name='softmax')(cnn) 
    model = Model(inputs=input_image, outputs=predict)