python制作caffe训练人脸关键点的HDF5_DATA代码

1. 参考博客

    通过修改caffe源代码来生成多标签训练数据的方式：

    https://blog.csdn.net/wwww1244/article/details/81034045

    https://blog.csdn.net/qq295456059/article/details/53142574

    通过python直接生成多标签训练数据hdf5的方式：

    https://www.cnblogs.com/frombeijingwithlove/p/5314042.html

2. 我修改的python代码来得到hdf5训练数据

import os
import random
import h5py
import numpy as np
from skimage import io
from skimage import transform as tf
trainpairlist=''
testpairlist=''
def save2hdf5(X,Y,filename):
    with h5py.File(filename, 'w') as f:
        #f['data'] = X
        #f['label'] = Y
        f.create_dataset('data', data=X)
        f.create_dataset('label', data=Y)
        print ('having saving a hdf5 file !')

def convert(source_path,pairlist,savepath,hdf5list,w,h):

    step = 5000
    
    fid=open(pairlist)
    lines= fid.readlines()
    fid.close()
    X=np.empty((step,3,w,h),dtype=np.float)
    Y=np.empty((step,14,1,1),dtype=np.float)
    i=0
    t=1
    #记得HDF5需要实现shuffle.
    #print ("list[0]", lines[0])
    #print ("list[1]", lines[1])

    #random.shuffle(x[, random])，用于将一个列表中的元素打乱
    random.shuffle(lines)
    for line in lines:
        words=line.split(' ')
        print("%d\n ", i)
        
        inputimage=words[0]
        #image  标签
        points = np.zeros((14,))
        
        points[0]=float(words[1])
        points[1]=float(words[2])
        points[2]=float(words[3])
        points[3]=float(words[4])
        points[4]=float(words[5])
        points[5]=float(words[6])
        points[6]=float(words[7])
        points[7]=float(words[8])
        points[8]=float(words[9])
        points[9]=float(words[10])
        points[10]=float(words[11])
        points[11]=float(words[12])
        points[12]=float(words[13])
        points[13]=float(words[14])
        
        im=io.imread(source_path+inputimage,as_grey=False)
        #im=tf.resize(im,(w,h))
        
        X[i,0,:,:]=im[:,:,0]
        X[i,1,:,:]=im[:,:,1]
        X[i,2,:,:]=im[:,:,2]
        Y[i,:,0,0]=points
        i = i + 1
        
        if i == step:
            filename = os.path.join(savepath, str(t) + '.h5')
            save2hdf5(X, Y, filename)
            with open(os.path.join(savepath, hdf5list), 'a') as f:
                f.write(filename + '\n')
            i = 0;
            t = t + 1

    if i > 0:
        filename = os.path.join(savepath, str(t) + '.h5')
        save2hdf5(X[0:i,:,:,:], Y[0:i,:,:,:], filename)
        with open(os.path.join(savepath, hdf5list), 'a') as f:
            f.write(filename + '\n')

if __name__=='__main__':
    w=40
    h=40
    source_path = 'G:/FacePoints/Transformed_image/'
    filelist = 'G:/FacePoints/train_trans.txt'
    save_path = 'G:/FacePoints/Transformed_image/'
    hdf5list='train_hdf.txt'
    convert(source_path, filelist, save_path, hdf5list, w, h)
    
    #source_path = '/media/crw/MyBook/MyDataset/FacePoint/test39X39/'
    #save_path = '/media/crw/MyBook/TrainData/HDF5/FacePoint/10000_39X39/test/'
    #hdf5list='/media/crw/MyBook/TrainData/HDF5/FacePoint/10000_39X39/test/test.txt'
    #filelist = '/media/crw/MyBook/MyDataset/FacePoint/test39X39/test.list'
    #convert(source_path,filelist,save_path,hdf5list,w,h)

因为一次将所有数据保存成.h5会报出内存的错误，所以修改代码，保存多段的.h5文件。

 

---

2019.2.12更新

---

因为训练出来的模型检测效果一般，最近在梳理问题的时候又发现这个博客https://blog.csdn.net/a1874738854/article/details/85073965#commentBox，其中指出python保存图像的格式与caffe的不一致，因此需要进行transpose，并且博客作者在这里将图片进行了去均值的预处理操作，参考这个代码将我的代码进行了更新更正。原作者的详细代码如下：

#影像文件夹所在目录
img_root = './image'
#训练数据txt路径
train_path = './train.txt'
#输出路径
train_out = './hdf5_train'
 
#将txt中的数据存入
with open(train_path) as f:
    lines = f.readlines()
 
file_list = []#存入影像路径
#建立标签和数据数组 
#若要生成hdf5数据，必须先把影像和标签变为数组
#本文标签数目为2，影像数据：channel = 3,width = 256,height = 256故生成如下形式数据
labels = np.zeros((len(lines), 2)).astype(np.int)
datas = np.zeros((len(lines),3,256,256)).astype(np.float32)
#读取数据
count = 0
for line in lines:
    file_list.append(line.split()[0])
    labels[count][0] = line.split()[1]
    labels[count][1] = line.split()[2]
    count += 1
f.close()
 
#caffe利用hfd5数据时，在输入层没有transform_param 参数，所以需要先对影像数据进行预处理
for i, file in enumerate(file_list):
    path = os.path.join(img_root,file)
    image = cv.imread(path)  #获取影像
    image = cv.resize(image,(256,256))#重采样为256*256大小的图像
    img = np.array(image)
    img = img.transpose(2,0,1)#讲图像从宽 高 通道 形式转化为通道 宽 高  caffe读取图像形式
    datas[i, :, :, :] = img.astype(np.float32)#hdf5要求数据为float或double形式
#获取影像均值
mean = datas.mean(axis=0)
mean = mean.mean(1).mean(1)
#将影像减去均值
for i in range(len(datas)):
    datas[i][0] = datas[i][0] - mean[0]
    datas[i][1] = datas[i][1] - mean[1]
    datas[i][2] = datas[i][2] - mean[2]
#保存hdf5文件  
with h5py.File(train_out,'w') as fout:
#'data'必须和train_val.prototxt文件里数据层中top：后边的名称一致，在修改prototxt文件时会进一步说明
    fout.create_dataset('data',data = datas)
    fout.create_dataset('label', data=labels)
fout.close()

我更正我自己后的代码如下：

import os
import random
import h5py
import numpy as np
from skimage import io

def save2hdf5(X,Y,filename):
    with h5py.File(filename, 'w') as f:
        #f['data'] = X
        #f['label'] = Y
        f.create_dataset('data', data=X)
        f.create_dataset('label', data=Y)
        print ('having saving a hdf5 file !')
    f.close()

def convert(source_path,pairlist,savepath,hdf5list,w,h):

    step = 5000
    
    fid=open(pairlist)
    lines= fid.readlines()
    fid.close()
    X=np.empty((step,3,w,h),dtype=np.float)
    Y=np.empty((step,14,1,1),dtype=np.float)
    i=0
    t=1
    #记得HDF5需要实现shuffle.
    #print ("list[0]", lines[0])
    #print ("list[1]", lines[1])

    for line in lines:
        words=line.split(' ')
        print("%d\n ", i)
        
        inputimage=words[0]
        #image  标签
        points = np.zeros((14,))
        
        points[0]=float(words[1])
        points[1]=float(words[2])
        points[2]=float(words[3])
        points[3]=float(words[4])
        points[4]=float(words[5])
        points[5]=float(words[6])
        points[6]=float(words[7])
        points[7]=float(words[8])
        points[8]=float(words[9])
        points[9]=float(words[10])
        points[10]=float(words[11])
        points[11]=float(words[12])
        points[12]=float(words[13])
        points[13]=float(words[14])
        
        im=io.imread(source_path+inputimage,as_grey=False)
        im=(im-127.5)/128    #图像归一化到(-1,1)
        #im=np.array(im)
        im=im.transpose(2, 0, 1)   #python图像存储格式为H*W*C，转换为caffe的格式即C*H*W
        #im=tf.resize(im,(w,h))
        X[i,:,:,:]=im.astype(np.float32)
        Y[i,:,0,0]=points
        i = i + 1
        
        if i == step:
            #获取影像均值
            #mean = X.mean(axis=0)
            #mean = mean.mean(1).mean(1)
            #将影像减去均值
            #for m in range(len(X)):
             #   X[m][0]=X[m][0]-mean[0]
              #  X[m][1]=X[m][1]-mean[1]
               # X[m][2]=X[m][2]-mean[2]
            filename = os.path.join(savepath, str(t) + '.h5')
            save2hdf5(X, Y, filename)
            with open(os.path.join(savepath, hdf5list), 'a') as f:
                f.write(filename + '\n')
            i = 0;
            t = t + 1

    if i > 0:
         #mean = X.mean(axis=0)
         #mean = mean.mean(1).mean(1)
         #for m in range(len(X)):
          #   X[m][0]=X[m][0]-mean[0]
           #  X[m][1]=X[m][1]-mean[1]
            # X[m][2]=X[m][2]-mean[2]

         filename = os.path.join(savepath, str(t) + '.h5')
         save2hdf5(X[0:i,:,:,:], Y[0:i,:,:,:], filename)
         with open(os.path.join(savepath, hdf5list), 'a') as f:
             f.write(filename + '\n')

if __name__=='__main__':
    w=40
    h=40
    source_path = 'G:/FacePoints/Transformed_image/'
    filelist = 'G:/FacePoints/train_trans.txt'
    save_path = 'G:/FacePoints/Transformed_image/'
    hdf5list='train_hdf.txt'
    convert(source_path, filelist, save_path, hdf5list, w, h)
    
    #source_path = '/media/crw/MyBook/MyDataset/FacePoint/test39X39/'
    #save_path = '/media/crw/MyBook/TrainData/HDF5/FacePoint/10000_39X39/test/'
    #hdf5list='/media/crw/MyBook/TrainData/HDF5/FacePoint/10000_39X39/test/test.txt'
    #filelist = '/media/crw/MyBook/MyDataset/FacePoint/test39X39/test.list'
    #convert(source_path,filelist,save_path,hdf5list,w,h)