caffe自定义模型训练的方法

1.准备数据集

共分为5类:

animal

flowers

plane

house

guitars

下载并进行解压

2.数据集进行分类

# coding:utf-8

import numpy as np
import os
import shutil

'''
    数据集的分类
    测试集: 300张
    训练集: 500张
'''

root_caffe = 'D:/Development/caffe/caffe-master/caffe-master/'

type = ['train','test']
data_type = ['animal', 'flower', 'guitar', 'house', 'plane']

#创建文件夹
model_dir = 'models/my_models_recognition/data/'

#数据集存放的路径
database_path = 'H:/database/'

#创建一个模型的总文件夹
path = root_caffe + model_dir
if not os.path.exists(path):
    os.makedirs(path)

for i in range(2):
    if not os.path.exists(path + type[i]):
        os.makedirs(path + type[i])

    for j in range(5):
        if not os.path.exists(path + type[i] + '/' + data_type[j]):
            os.makedirs(path + type[i] + '/' + data_type[j])

        # 读取文件
        for root,dir,file in os.walk(database_path + data_type[j]):
            print(len(file))
            paths = []
            if(i == 0): # 训练集
                list = file[0:500]

                for k in range(500):
                    print(database_path + data_type[j] + '/'+ list[k])
                    print(path + type[i] + '/' + data_type[j])
                    shutil.copyfile(database_path + data_type[j] + '/'+ list[k]  , #图片的原始路径
                                    path + type[i] + '/' + data_type[j] + '/' + list[k]  ) #图片的caffe模型路径

            elif(i==1): # 测试集
                list = file[0:300]

                for k in range(300):
                    print(database_path + data_type[j] + '/' + list[k])
                    print(path + type[i] + '/' + data_type[j])
                    shutil.copyfile(database_path + data_type[j] + '/' + list[k],
                                    path + type[i] + '/' + data_type[j] + '/' + list[k])

3.标签文件的制作

# coding: utf-8

import os  


# In[2]:

#定义Caffe根目录
caffe_root = 'D:/Development/caffe/caffe-master/caffe-master/'

#制作训练标签数据
i = 0 #标签
with open(caffe_root + 'models/my_models_recognition/labels/train.txt','wb') as train_txt:
    for root,dirs,files in os.walk(caffe_root+'models/my_models_recognition/data/train/'): #遍历文件夹
        for dir in dirs:
            for root,dirs,files in os.walk(caffe_root+'models/my_models_recognition/data/train/'+str(dir)): #遍历每一个文件夹中的文件 
                for file in files:
                    image_file = str(dir) + '\\' + str(file)
                    label = image_file + ' ' + str(i) + '\n'       #文件路径+空格+标签编号+换行 
                    train_txt.writelines(label)                   #写入标签文件中
                i+=1 #编号加1


#制作测试标签数据
i=0 #标签
with open(caffe_root + 'models/my_models_recognition/labels/test.txt','wb') as test_txt:
    for root,dirs,files in os.walk(caffe_root+'models/my_models_recognition/data/test/'): #遍历文件夹
        for dir in dirs:
            for root,dirs,files in os.walk(caffe_root+'models/my_models_recognition/data/test/'+str(dir)): #遍历每一个文件夹中的文件
                for file in files:
                    image_file = str(dir) + '\\' + str(file)
                    label = image_file + ' ' + str(i) + '\n'       #文件路径+空格+标签编号+换行 
                    test_txt.writelines(label)                   #写入标签文件中
                i+=1#编号加1
                
print "成功生成文件列表" 

4.数据转化，把图片转化为LMDB格式

convert_imageset_train.bat：

%格式转化的可执行文件%
%重新设定文件大小%
%打乱图片%
%转化格式%
%图片路径%
%图片标签%
%lmdb文件的输出路径%

SET GLOG_logtostderr=1
D:\Development\caffe\caffe-master\caffe-master\Build\x64\Release\convert_imageset.exe ^
--resize_height=256  --resize_width=256 ^
--shuffle ^
--backend="lmdb" ^
D:\Development\caffe\caffe-master\caffe-master\models\my_models_recognition\data\train\ ^
D:\Development\caffe\caffe-master\caffe-master\models\my_models_recognition\labels\train.txt ^
D:\Development\caffe\caffe-master\caffe-master\models\my_models_recognition\lmdb\train\

pause

convert_imageset_test.bat：

%格式转化的可执行文件%
%重新设定文件大小%
%打乱图片%
%转化格式%
%图片路径%
%图片标签%
%lmdb文件的输出路径%


D:\Development\caffe\caffe-master\caffe-master\Build\x64\Release\convert_imageset.exe ^
--resize_height=256  --resize_width=256 ^
--shuffle ^
--backend="lmdb" ^
D:\Development\caffe\caffe-master\caffe-master\models\my_models_recognition\data\test\ ^
D:\Development\caffe\caffe-master\caffe-master\models\my_models_recognition\labels\test.txt ^
D:\Development\caffe\caffe-master\caffe-master\models\my_models_recognition\lmdb\test\

pause

5.编写网络

用的是caffenet的网络，对此网络进行修改

train_val.prototxt:

name: "CaffeNet"
layer {
  name: "data"
  type: "Data"
  top: "data"
  top: "label"
  include {
    phase: TRAIN
  }
#  transform_param { #求均值方法1
#    mirror: true
#    crop_size: 227
#    mean_file: "data/ilsvrc12/imagenet_mean.binaryproto"
#  }

#mean pixel / channel-wise mean instead of mean image
  transform_param { #求均值方法2
    crop_size: 227  #随机截取227*227
    mean_value: 104 #大量图片求的均值
    mean_value: 117
    mean_value: 123
    mirror: true
  }
  data_param {
    source: "D:/Development/caffe/caffe-master/caffe-master/models/my_models_recognition/lmdb/train"
    batch_size: 50
    backend: LMDB
  }
}
layer {
  name: "data"
  type: "Data"
  top: "data"
  top: "label"
  include {
    phase: TEST
  }
#  transform_param {
#    mirror: false
#    crop_size: 227
#    mean_file: "data/ilsvrc12/imagenet_mean.binaryproto"
#  }

# mean pixel / channel-wise mean instead of mean image
  transform_param {
    crop_size: 227
    mean_value: 104
    mean_value: 117
    mean_value: 123
    mirror: false
  }
  data_param {
    source: "D:/Development/caffe/caffe-master/caffe-master/models/my_models_recognition/lmdb/test"
    batch_size: 50
    backend: LMDB
  }
}
layer {
  name: "conv1"
  type: "Convolution"
  bottom: "data"
  top: "conv1"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  convolution_param {
    num_output: 96
    kernel_size: 11
    stride: 4
    weight_filler {
      type: "gaussian"
      std: 0.01
    }
    bias_filler {
      type: "constant"
      value: 0
    }
  }
}
layer {
  name: "relu1"
  type: "ReLU"
  bottom: "conv1"
  top: "conv1"
}
layer {
  name: "pool1"
  type: "Pooling"
  bottom: "conv1"
  top: "pool1"
  pooling_param {
    pool: MAX
    kernel_size: 3
    stride: 2
  }
}
layer {
  name: "norm1"
  type: "LRN"
  bottom: "pool1"
  top: "norm1"
  lrn_param {
    local_size: 5
    alpha: 0.0001
    beta: 0.75
  }
}
layer {
  name: "conv2"
  type: "Convolution"
  bottom: "norm1"
  top: "conv2"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  convolution_param {
    num_output: 256
    pad: 2
    kernel_size: 5
    group: 2
    weight_filler {
      type: "gaussian"
      std: 0.01
    }
    bias_filler {
      type: "constant"
      value: 1
    }
  }
}
layer {
  name: "relu2"
  type: "ReLU"
  bottom: "conv2"
  top: "conv2"
}
layer {
  name: "pool2"
  type: "Pooling"
  bottom: "conv2"
  top: "pool2"
  pooling_param {
    pool: MAX
    kernel_size: 3
    stride: 2
  }
}
layer {
  name: "norm2"
  type: "LRN"
  bottom: "pool2"
  top: "norm2"
  lrn_param {
    local_size: 5
    alpha: 0.0001
    beta: 0.75
  }
}
layer {
  name: "conv3"
  type: "Convolution"
  bottom: "norm2"
  top: "conv3"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  convolution_param {
    num_output: 384
    pad: 1
    kernel_size: 3
    weight_filler {
      type: "gaussian"
      std: 0.01
    }
    bias_filler {
      type: "constant"
      value: 0
    }
  }
}
layer {
  name: "relu3"
  type: "ReLU"
  bottom: "conv3"
  top: "conv3"
}
layer {
  name: "conv4"
  type: "Convolution"
  bottom: "conv3"
  top: "conv4"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  convolution_param {
    num_output: 384
    pad: 1
    kernel_size: 3
    group: 2
    weight_filler {
      type: "gaussian"
      std: 0.01
    }
    bias_filler {
      type: "constant"
      value: 1
    }
  }
}
layer {
  name: "relu4"
  type: "ReLU"
  bottom: "conv4"
  top: "conv4"
}
layer {
  name: "conv5"
  type: "Convolution"
  bottom: "conv4"
  top: "conv5"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  convolution_param {
    num_output: 256
    pad: 1
    kernel_size: 3
    group: 2
    weight_filler {
      type: "gaussian"
      std: 0.01
    }
    bias_filler {
      type: "constant"
      value: 1
    }
  }
}
layer {
  name: "relu5"
  type: "ReLU"
  bottom: "conv5"
  top: "conv5"
}
layer {
  name: "pool5"
  type: "Pooling"
  bottom: "conv5"
  top: "pool5"
  pooling_param {
    pool: MAX
    kernel_size: 3
    stride: 2
  }
}
layer {
  name: "fc6"
  type: "InnerProduct"
  bottom: "pool5"
  top: "fc6"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  inner_product_param {
    num_output: 512
    weight_filler {
      type: "gaussian"
      std: 0.005
    }
    bias_filler {
      type: "constant"
      value: 1
    }
  }
}
layer {
  name: "relu6"
  type: "ReLU"
  bottom: "fc6"
  top: "fc6"
}
layer {
  name: "drop6"
  type: "Dropout"
  bottom: "fc6"
  top: "fc6"
  dropout_param {
    dropout_ratio: 0.5
  }
}
layer {
  name: "fc7"
  type: "InnerProduct"
  bottom: "fc6"
  top: "fc7"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  inner_product_param {
    num_output: 512
    weight_filler {
      type: "gaussian"
      std: 0.005
    }
    bias_filler {
      type: "constant"
      value: 1
    }
  }
}
layer {
  name: "relu7"
  type: "ReLU"
  bottom: "fc7"
  top: "fc7"
}
layer {
  name: "drop7"
  type: "Dropout"
  bottom: "fc7"
  top: "fc7"
  dropout_param {
    dropout_ratio: 0.5
  }
}
layer {
  name: "fc8"
  type: "InnerProduct"
  bottom: "fc7"
  top: "fc8"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 2
    decay_mult: 0
  }
  inner_product_param {
    num_output: 5
    weight_filler {
      type: "gaussian"
      std: 0.01
    }
    bias_filler {
      type: "constant"
      value: 0
    }
  }
}
layer {
  name: "accuracy"
  type: "Accuracy"
  bottom: "fc8"
  bottom: "label"
  top: "accuracy"
  include {
    phase: TEST
  }
}
layer {
  name: "loss"
  type: "SoftmaxWithLoss"
  bottom: "fc8"
  bottom: "label"
  top: "loss"
}

6.修改超参数文件

solver.prototxt:

net: "D:/Development/caffe/caffe-master/caffe-master/models/my_models_recognition/train_val.prototxt"
test_iter: 30 #1000 /batch
test_interval: 200  #间隔数

base_lr: 0.01
lr_policy: "step"  #每隔stepsize步，对学习率进行改变
gamma: 0.1
stepsize: 1000

display: 100    #每100次显示一次
max_iter: 5000
momentum: 0.9
weight_decay: 0.0005
snapshot: 5000
snapshot_prefix: "D:/Development/caffe/caffe-master/caffe-master/models/my_models_recognition/model"
solver_mode: CPU

7.编写train的bat文件

 %train训练数据%
 %超参数文件%

D:\Development\caffe\caffe-master\caffe-master\Build\x64\Release\caffe.exe train ^
-solver=D:/Development/caffe/caffe-master/caffe-master/models/my_models_recognition/solver.prototxt

pause

这里是训练5000次 每1000次打印一次，我这里只是训练5000次之后，对文件进行保存:

8.进行测试

      8.1 编写网络文件

name: "CaffeNet"
layer {
  name: "data"
  type: "Input"
  top: "data"
  input_param { shape: { dim: 1 dim: 3 dim: 227 dim: 227 } }	 #dim:1 修改
}
layer {
  name: "conv1"
  type: "Convolution"
  bottom: "data"
  top: "conv1"
  convolution_param {
    num_output: 96
    kernel_size: 11
    stride: 4
  }
}
layer {
  name: "relu1"
  type: "ReLU"
  bottom: "conv1"
  top: "conv1"
}
layer {
  name: "pool1"
  type: "Pooling"
  bottom: "conv1"
  top: "pool1"
  pooling_param {
    pool: MAX
    kernel_size: 3
    stride: 2
  }
}
layer {
  name: "norm1"
  type: "LRN"
  bottom: "pool1"
  top: "norm1"
  lrn_param {
    local_size: 5
    alpha: 0.0001
    beta: 0.75
  }
}
layer {
  name: "conv2"
  type: "Convolution"
  bottom: "norm1"
  top: "conv2"
  convolution_param {
    num_output: 256
    pad: 2
    kernel_size: 5
    group: 2
  }
}
layer {
  name: "relu2"
  type: "ReLU"
  bottom: "conv2"
  top: "conv2"
}
layer {
  name: "pool2"
  type: "Pooling"
  bottom: "conv2"
  top: "pool2"
  pooling_param {
    pool: MAX
    kernel_size: 3
    stride: 2
  }
}
layer {
  name: "norm2"
  type: "LRN"
  bottom: "pool2"
  top: "norm2"
  lrn_param {
    local_size: 5
    alpha: 0.0001
    beta: 0.75
  }
}
layer {
  name: "conv3"
  type: "Convolution"
  bottom: "norm2"
  top: "conv3"
  convolution_param {
    num_output: 384
    pad: 1
    kernel_size: 3
  }
}
layer {
  name: "relu3"
  type: "ReLU"
  bottom: "conv3"
  top: "conv3"
}
layer {
  name: "conv4"
  type: "Convolution"
  bottom: "conv3"
  top: "conv4"
  convolution_param {
    num_output: 384
    pad: 1
    kernel_size: 3
    group: 2
  }
}
layer {
  name: "relu4"
  type: "ReLU"
  bottom: "conv4"
  top: "conv4"
}
layer {
  name: "conv5"
  type: "Convolution"
  bottom: "conv4"
  top: "conv5"
  convolution_param {
    num_output: 256
    pad: 1
    kernel_size: 3
    group: 2
  }
}
layer {
  name: "relu5"
  type: "ReLU"
  bottom: "conv5"
  top: "conv5"
}
layer {
  name: "pool5"
  type: "Pooling"
  bottom: "conv5"
  top: "pool5"
  pooling_param {
    pool: MAX
    kernel_size: 3
    stride: 2
  }
}
layer {
  name: "fc6"
  type: "InnerProduct"
  bottom: "pool5"
  top: "fc6"
  inner_product_param {
    num_output: 512  #修改，原因自己电脑的显存或cpu有限
  }
}
layer {
  name: "relu6"
  type: "ReLU"
  bottom: "fc6"
  top: "fc6"
}
layer {
  name: "drop6"
  type: "Dropout"
  bottom: "fc6"
  top: "fc6"
  dropout_param {
    dropout_ratio: 0.5
  }
}
layer {
  name: "fc7"
  type: "InnerProduct"
  bottom: "fc6"
  top: "fc7"
  inner_product_param {
    num_output: 512	  #修改，原因自己电脑的显存或cpu有限
  }
}
layer {
  name: "relu7"
  type: "ReLU"
  bottom: "fc7"
  top: "fc7"
}
layer {
  name: "drop7"
  type: "Dropout"
  bottom: "fc7"
  top: "fc7"
  dropout_param {
    dropout_ratio: 0.5
  }
}
layer {
  name: "fc8"
  type: "InnerProduct"
  bottom: "fc7"
  top: "fc8"
  inner_product_param {
    num_output: 5   #修改类别
  }
}
layer {
  name: "prob"
  type: "Softmax"
  bottom: "fc8"
  top: "prob"
}

      8.2 找训练图片

               在数据集中每样寻找几张

  8.3编写python文件测试

import caffe
import numpy as np
import matplotlib.pyplot as plt
import os
import PIL
from PIL import Image
import sys


#定义Caffe根目录  这个需要根据自己的目录进行修改
caffe_root = 'D:/Development/caffe/caffe-master/caffe-master/'
#网络结构描述文件
deploy_file = caffe_root+'models/my_models_recognition/deploy.prototxt'
#训练好的模型
model_file = caffe_root+'models/my_models_recognition/model/model_iter_5000.caffemodel'

#gpu模式
#caffe.set_device(0)
caffe.set_mode_cpu()

#定义网络模型
net = caffe.Classifier(deploy_file, #调用deploy文件
                       model_file,  #调用模型文件
                       channel_swap=(2,1,0),  #caffe中图片是BGR格式，而原始格式是RGB，所以要转化
                       raw_scale=255,         #python中将图片存储为[0, 1]，而caffe中将图片存储为[0, 255]，所以需要一个转换
                       image_dims=(227, 227)) #输入模型的图片要是227*227的图片


#分类标签文件
imagenet_labels_filename = caffe_root +'models/my_models_recognition/labels/label.txt'
#载入分类标签文件
labels = np.loadtxt(imagenet_labels_filename, str)

#对目标路径中的图像，遍历并分类
for root,dirs,files in os.walk(caffe_root+'models/my_models_recognition/image/'): #循环每一个文件
    for file in files:
        #加载要分类的图片
        image_file = os.path.join(root,file)
        input_image = caffe.io.load_image(image_file)

        #打印图片路径及名称
        image_path = os.path.join(root,file)
        print(image_path)
        
        #显示图片
        img=Image.open(image_path)
        plt.imshow(img)
        plt.axis('off')
        plt.show()
        
        #预测图片类别
        prediction = net.predict([input_image])
        print 'predicted class:',prediction[0].argmax()

        # 输出概率最大的前5个预测结果
        top_k = prediction[0].argsort()[::-1]
        for node_id in top_k:     
            #获取分类名称
            human_string = labels[node_id]
            #获取该分类的置信度
            score = prediction[0][node_id]
            print('%s (score = %.5f)' % (human_string, score))

8.4 测试结果: