1 img–>lmdb
firstly we need the train.txt,and test.txt
import os
path='/path/path/'
#we can also use os.walk(dir)
def imgname_to_txt(dir,output_txt):
output=open(output_txt,mode='w')#mode:write read add
files=os.listdir(dir)
for name in files:
fullname=os.path.join(dir,name)
if (os.path.isdir(fullname)):
subfiles=os.listdir(fullname)
for subname in subfiles:
output.write(name+'/'+subname+' '+fullname[-1]+'\n')
else:
output.write(name+' '+'0'+'\n')
output.close()
imgname_to_txt(path+'/img/train',path+'/data/train.txt')
imgname_to_txt(path+'/img/test',path+'/data/test.txt')
and maybe we want a val data set:
import numpy as np
train=open(path+'/data/train.txt','r')
train_lines=train.readlines()
l=len(train_lines)
print l
train.close()
train_sub=open(path+'/data/train_sub.txt','w')
val=open(path+'/data/val.txt','w')
np.random.seed(0)
randomval=np.random.choice(range(l),int(l*0.3),replace=False)#replace=False !
print len(randomval)
print randomval[:5]
for i in randomval:
val.write(train_lines[i])
val.close()
train_index=[i for i in range(l) if i not in randomval]
print train_index[:5]
print len(train_index)
for j in train_index:
train_sub.write(train_lines[j])
train_sub.close()
let convert img to lmdb with $TOOLS/convert_imageset, with the sh file in imagenet :
echo "Creating train lmdb..."
GLOG_logtostderr=1 $TOOLS/convert_imageset \
--resize_height=$RESIZE_HEIGHT \
--resize_width=$RESIZE_WIDTH \
--shuffle \
$TRAIN_DATA_ROOT \
$DATA/train_sub.txt \
$EXAMPLE/driver_train_sub_lmdb
we need to compute image mean:
$TOOLS/compute_image_mean $EXAMPLE/**_train_lmdb \
$DATA/imagenet_mean.binaryproto
draw_net
./python/draw_net.py --rankdir TB /media/beatree/file/note_for_project/work/conpetitions/kaggle/driver/dirver_predict_img.prototxt /media/beatree/file/note_for_project/work/conpetitions/kaggle/driver/driver_caffnet2.png
2 train
./build/tools/caffe train -solver /media/beatree/file/note_for_project/work/conpetitions/kaggle/driver/solverall.prototxt -weights /home/beatree/caffe-rc3/models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel -gpu 0
3 predict
3.1 binaryproto->npy
http://blog.csdn.net/hyman_yx/article/details/51732656
import caffe
import numpy as np
MEAN_PROTO_PATH = 'mean.binaryproto' # 待转换的pb格式图像均值文件路径
MEAN_NPY_PATH = 'mean.npy' # 转换后的numpy格式图像均值文件路径
blob = caffe.proto.caffe_pb2.BlobProto() # 创建protobuf blob
data = open(MEAN_PROTO_PATH, 'rb' ).read() # 读入mean.binaryproto文件内容
blob.ParseFromString(data) # 解析文件内容到blob
array = np.array(caffe.io.blobproto_to_array(blob))# 将blob中的均值转换成numpy格式,array的shape (mean_number,channel, hight, width)
mean_npy = array.mean(0)
np.save(MEAN_NPY_PATH ,mean_npy)
# print mean_npy
3.2 classification
mport numpy as np
import matplotlib.pyplot as plt
# display plots in this notebook
%matplotlib inline
# set display defaults
plt.rcParams['figure.figsize'] = (10, 10) # large images
plt.rcParams['image.interpolation'] = 'nearest' # don't interpolate: show square pixels
plt.rcParams['image.cmap'] = 'gray' # use grayscale output rather than a (potentially misleading) color heatmap
path='/pathpath/driver/'
import caffe
caffe.set_device(0)
caffe.set_mode_gpu()
model_def = path + 'dirver_predict_img.prototxt'
model_weights = path+'model/finetune_iter_20000.caffemodel'
net = caffe.Net(model_def, # defines the structure of the model
model_weights, # contains the trained weights
caffe.TEST) # use test mode (e.g., don't perform dropout)
# load the mean ImageNet image (as distributed with Caffe) for subtraction
mu = np.load(path + 'data/mean.npy')
mu = mu.mean(1).mean(1) # average over pixels to obtain the mean (BGR) pixel values
print 'mean-subtracted values:', zip('BGR', mu)
# create transformer for the input called 'data'
transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})
transformer.set_transpose('data', (2,0,1)) # move image channels to outermost dimension
transformer.set_mean('data', mu) # subtract the dataset-mean value in each channel
transformer.set_raw_scale('data', 255) # rescale from [0, 1] to [0, 255]
transformer.set_channel_swap('data', (2,1,0)) # swap channels from RGB to BGR
# set the size of the input (we can skip this if we're happy
# with the default; we can also change it later, e.g., for different batch sizes)
net.blobs['data'].reshape(32, # batch size
3, # 3-channel (BGR) images
227, 227) # image size is 227
import pandas as pd
import numpy as np
sample_df=pd.read_csv(path+'sample_submission.csv')
submission_df=pd.DataFrame(columns=sample_df.columns,index=sample_df.index)
submission_df.img=sample_df.img
batch_size=32
start_index=0
sample_len=len(sample_df)
while start_index<sample_len:
end_index=min(start_index+batch_size,sample_len)
print 'predicting',start_index,':',end_index
img=[caffe.io.load_image(path+'img/test/{}'.format(sample_df.img[i]))for i in range(start_index,end_index)]
transformed_image =[ transformer.preprocess('data', img[i])for i in range (len(img)) ]
if end_index<sample_len:
net.blobs['data'].data[0:32]=transformed_image
else:
net.blobs['data'].data[0:(end_index%32)] = transformed_image
output = net.forward()
if end_index<sample_len:
output_prob = output['probs'] [0:32]
else:
output_prob = output['probs'] [0:(end_index%32)]
print output_prob.argmax(1)
submission_df.iloc[range(start_index,end_index),1:]=output_prob
start_index=end_index
submission_df.to_csv(path+'submission.csv',index=False)
print 'done'
result