1. 参考博客
通过修改caffe源代码来生成多标签训练数据的方式:
https://blog.csdn.net/wwww1244/article/details/81034045
https://blog.csdn.net/qq295456059/article/details/53142574
通过python直接生成多标签训练数据hdf5的方式:
https://www.cnblogs.com/frombeijingwithlove/p/5314042.html
2. 我修改的python代码来得到hdf5训练数据
import os
import random
import h5py
import numpy as np
from skimage import io
from skimage import transform as tf
trainpairlist=''
testpairlist=''
def save2hdf5(X,Y,filename):
with h5py.File(filename, 'w') as f:
#f['data'] = X
#f['label'] = Y
f.create_dataset('data', data=X)
f.create_dataset('label', data=Y)
print ('having saving a hdf5 file !')
def convert(source_path,pairlist,savepath,hdf5list,w,h):
step = 5000
fid=open(pairlist)
lines= fid.readlines()
fid.close()
X=np.empty((step,3,w,h),dtype=np.float)
Y=np.empty((step,14,1,1),dtype=np.float)
i=0
t=1
#记得HDF5需要实现shuffle.
#print ("list[0]", lines[0])
#print ("list[1]", lines[1])
#random.shuffle(x[, random]),用于将一个列表中的元素打乱
random.shuffle(lines)
for line in lines:
words=line.split(' ')
print("%d\n ", i)
inputimage=words[0]
#image 标签
points = np.zeros((14,))
points[0]=float(words[1])
points[1]=float(words[2])
points[2]=float(words[3])
points[3]=float(words[4])
points[4]=float(words[5])
points[5]=float(words[6])
points[6]=float(words[7])
points[7]=float(words[8])
points[8]=float(words[9])
points[9]=float(words[10])
points[10]=float(words[11])
points[11]=float(words[12])
points[12]=float(words[13])
points[13]=float(words[14])
im=io.imread(source_path+inputimage,as_grey=False)
#im=tf.resize(im,(w,h))
X[i,0,:,:]=im[:,:,0]
X[i,1,:,:]=im[:,:,1]
X[i,2,:,:]=im[:,:,2]
Y[i,:,0,0]=points
i = i + 1
if i == step:
filename = os.path.join(savepath, str(t) + '.h5')
save2hdf5(X, Y, filename)
with open(os.path.join(savepath, hdf5list), 'a') as f:
f.write(filename + '\n')
i = 0;
t = t + 1
if i > 0:
filename = os.path.join(savepath, str(t) + '.h5')
save2hdf5(X[0:i,:,:,:], Y[0:i,:,:,:], filename)
with open(os.path.join(savepath, hdf5list), 'a') as f:
f.write(filename + '\n')
if __name__=='__main__':
w=40
h=40
source_path = 'G:/FacePoints/Transformed_image/'
filelist = 'G:/FacePoints/train_trans.txt'
save_path = 'G:/FacePoints/Transformed_image/'
hdf5list='train_hdf.txt'
convert(source_path, filelist, save_path, hdf5list, w, h)
#source_path = '/media/crw/MyBook/MyDataset/FacePoint/test39X39/'
#save_path = '/media/crw/MyBook/TrainData/HDF5/FacePoint/10000_39X39/test/'
#hdf5list='/media/crw/MyBook/TrainData/HDF5/FacePoint/10000_39X39/test/test.txt'
#filelist = '/media/crw/MyBook/MyDataset/FacePoint/test39X39/test.list'
#convert(source_path,filelist,save_path,hdf5list,w,h)
因为一次将所有数据保存成.h5会报出内存的错误,所以修改代码,保存多段的.h5文件。
2019.2.12更新
因为训练出来的模型检测效果一般,最近在梳理问题的时候又发现这个博客https://blog.csdn.net/a1874738854/article/details/85073965#commentBox,其中指出python保存图像的格式与caffe的不一致,因此需要进行transpose,并且博客作者在这里将图片进行了去均值的预处理操作,参考这个代码将我的代码进行了更新更正。原作者的详细代码如下:
#影像文件夹所在目录
img_root = './image'
#训练数据txt路径
train_path = './train.txt'
#输出路径
train_out = './hdf5_train'
#将txt中的数据存入
with open(train_path) as f:
lines = f.readlines()
file_list = []#存入影像路径
#建立标签和数据数组
#若要生成hdf5数据,必须先把影像和标签变为数组
#本文标签数目为2,影像数据:channel = 3,width = 256,height = 256故生成如下形式数据
labels = np.zeros((len(lines), 2)).astype(np.int)
datas = np.zeros((len(lines),3,256,256)).astype(np.float32)
#读取数据
count = 0
for line in lines:
file_list.append(line.split()[0])
labels[count][0] = line.split()[1]
labels[count][1] = line.split()[2]
count += 1
f.close()
#caffe利用hfd5数据时,在输入层没有transform_param 参数,所以需要先对影像数据进行预处理
for i, file in enumerate(file_list):
path = os.path.join(img_root,file)
image = cv.imread(path) #获取影像
image = cv.resize(image,(256,256))#重采样为256*256大小的图像
img = np.array(image)
img = img.transpose(2,0,1)#讲图像从宽 高 通道 形式转化为通道 宽 高 caffe读取图像形式
datas[i, :, :, :] = img.astype(np.float32)#hdf5要求数据为float或double形式
#获取影像均值
mean = datas.mean(axis=0)
mean = mean.mean(1).mean(1)
#将影像减去均值
for i in range(len(datas)):
datas[i][0] = datas[i][0] - mean[0]
datas[i][1] = datas[i][1] - mean[1]
datas[i][2] = datas[i][2] - mean[2]
#保存hdf5文件
with h5py.File(train_out,'w') as fout:
#'data'必须和train_val.prototxt文件里数据层中top:后边的名称一致,在修改prototxt文件时会进一步说明
fout.create_dataset('data',data = datas)
fout.create_dataset('label', data=labels)
fout.close()
我更正我自己后的代码如下:
import os
import random
import h5py
import numpy as np
from skimage import io
def save2hdf5(X,Y,filename):
with h5py.File(filename, 'w') as f:
#f['data'] = X
#f['label'] = Y
f.create_dataset('data', data=X)
f.create_dataset('label', data=Y)
print ('having saving a hdf5 file !')
f.close()
def convert(source_path,pairlist,savepath,hdf5list,w,h):
step = 5000
fid=open(pairlist)
lines= fid.readlines()
fid.close()
X=np.empty((step,3,w,h),dtype=np.float)
Y=np.empty((step,14,1,1),dtype=np.float)
i=0
t=1
#记得HDF5需要实现shuffle.
#print ("list[0]", lines[0])
#print ("list[1]", lines[1])
for line in lines:
words=line.split(' ')
print("%d\n ", i)
inputimage=words[0]
#image 标签
points = np.zeros((14,))
points[0]=float(words[1])
points[1]=float(words[2])
points[2]=float(words[3])
points[3]=float(words[4])
points[4]=float(words[5])
points[5]=float(words[6])
points[6]=float(words[7])
points[7]=float(words[8])
points[8]=float(words[9])
points[9]=float(words[10])
points[10]=float(words[11])
points[11]=float(words[12])
points[12]=float(words[13])
points[13]=float(words[14])
im=io.imread(source_path+inputimage,as_grey=False)
im=(im-127.5)/128 #图像归一化到(-1,1)
#im=np.array(im)
im=im.transpose(2, 0, 1) #python图像存储格式为H*W*C,转换为caffe的格式即C*H*W
#im=tf.resize(im,(w,h))
X[i,:,:,:]=im.astype(np.float32)
Y[i,:,0,0]=points
i = i + 1
if i == step:
#获取影像均值
#mean = X.mean(axis=0)
#mean = mean.mean(1).mean(1)
#将影像减去均值
#for m in range(len(X)):
# X[m][0]=X[m][0]-mean[0]
# X[m][1]=X[m][1]-mean[1]
# X[m][2]=X[m][2]-mean[2]
filename = os.path.join(savepath, str(t) + '.h5')
save2hdf5(X, Y, filename)
with open(os.path.join(savepath, hdf5list), 'a') as f:
f.write(filename + '\n')
i = 0;
t = t + 1
if i > 0:
#mean = X.mean(axis=0)
#mean = mean.mean(1).mean(1)
#for m in range(len(X)):
# X[m][0]=X[m][0]-mean[0]
# X[m][1]=X[m][1]-mean[1]
# X[m][2]=X[m][2]-mean[2]
filename = os.path.join(savepath, str(t) + '.h5')
save2hdf5(X[0:i,:,:,:], Y[0:i,:,:,:], filename)
with open(os.path.join(savepath, hdf5list), 'a') as f:
f.write(filename + '\n')
if __name__=='__main__':
w=40
h=40
source_path = 'G:/FacePoints/Transformed_image/'
filelist = 'G:/FacePoints/train_trans.txt'
save_path = 'G:/FacePoints/Transformed_image/'
hdf5list='train_hdf.txt'
convert(source_path, filelist, save_path, hdf5list, w, h)
#source_path = '/media/crw/MyBook/MyDataset/FacePoint/test39X39/'
#save_path = '/media/crw/MyBook/TrainData/HDF5/FacePoint/10000_39X39/test/'
#hdf5list='/media/crw/MyBook/TrainData/HDF5/FacePoint/10000_39X39/test/test.txt'
#filelist = '/media/crw/MyBook/MyDataset/FacePoint/test39X39/test.list'
#convert(source_path,filelist,save_path,hdf5list,w,h)