该模块的功能主要是生成rpn proposals。
+ cv2.imread读取的图像的存储格式为H W K,且三通道的顺序为BGR
+ PIL.Image.open读取图片的存储格式为:W H (此时的数据为PIL库的某种对象),且三通道顺序为RGB
但是,当转换为np.array的时候,存储格式将H W K,常用的代码段为:
def load_image(self, idx):
"""
Load input image and preprocess for Caffe:
- cast to float
- switch channels RGB -> BGR
- subtract mean
- transpose to channel x height x width order
"""
im = Image.open('xxx.jpg')
in_ = np.array(im, dtype=np.float32)
in_ = in_[:,:,::-1]
in_ -= self.mean
in_ = in_.transpose((2,0,1))
return in_
def imdb_proposals(net, imdb):
def imdb_proposals(net, imdb):
"""Generate RPN proposals on all images in an imdb."""
_t = Timer()
imdb_boxes = [[] for _ in xrange(imdb.num_images)]
for i in xrange(imdb.num_images):
# cv2.imread读取的图像的存储格式为H W K,且三通道的顺序为BGR
# PIL.Image.open读取图片的存储格式为:W H K ,且三通道顺序为RGB
im = cv2.imread(imdb.image_path_at(i))
_t.tic()
#调用 im_proposals生成单张图片的rpn proposals, 以及得分
imdb_boxes[i], scores = im_proposals(net, im)
_t.toc()
print 'im_proposals: {:d}/{:d} {:.3f}s' \
.format(i + 1, imdb.num_images, _t.average_time)
if 0:
dets = np.hstack((imdb_boxes[i], scores))
# from IPython import embed; embed()
_vis_proposals(im, dets[:3, :], thresh=0.9)
plt.show()
return imdb_boxes
def im_proposals(net, im):该方法中会调用网络的forwad,从而得到想要的boxes和scores
def im_proposals(net, im):
"""Generate RPN proposals on a single image."""
blobs = {}
#调用_get_image_blob函数将图像转换为caffe所支持的输入数据结构即四维blob
blobs['data'], blobs['im_info'] = _get_image_blob(im)
# *(blobs['data'].shape 中的 ‘*’涉及到了python中函数的参数收集及逆操作
net.blobs['data'].reshape(*(blobs['data'].shape))
net.blobs['im_info'].reshape(*(blobs['im_info'].shape))
blobs_out = net.forward(
data=blobs['data'].astype(np.float32, copy=False),
im_info=blobs['im_info'].astype(np.float32, copy=False))
#返回im_info这个blob中存储的scale,等价于blobs['im_info'][0][2]
scale = blobs['im_info'][0, 2]
# 通过net的前向传播得到boxes, scores,注意,将boxes返回之前,需要将其缩放会原来的size
# blobs_out['rois'][:]的第一位为类别,后四位才是坐标
boxes = blobs_out['rois'][:, 1:].copy() / scale
scores = blobs_out['scores'].copy()
return boxes, scores
下面重点讲一下blobs_out = net.forward(data=blobs['data'].astype(np.float32, copy=False), im_info=blobs['im_info'].astype(np.float32, copy=False))
, 当然纯属个人观点:
def _Net_forward(self, blobs=None, start=None, end=None, **kwargs):
"""
Forward pass: prepare inputs and run the net forward.
Parameters
----------
blobs : list of blobs to return in addition to output blobs.
kwargs : Keys are input blob names and values are blob ndarrays.
For formatting inputs for Caffe, see Net.preprocess().
If None, input is taken from data layers.
start : optional name of layer at which to begin the forward pass
end : optional name of layer at which to finish the forward pass
(inclusive)
Returns
-------
outs : {blob name: blob ndarray} dict.
"""
if blobs is None:
blobs = []
# 返回name为start的layer的id,作为start_ind
start_ind = list(self._layer_names).index(start)
else:
start_ind = 0
if end is not None:
end_ind = list(self._layer_names).index(end)
outputs = set([end] + blobs)
else:
end_ind = len(self.layers) - 1
outputs = set(self.outputs + blobs)
if kwargs:
if set(kwargs.keys()) != set(self.inputs):
raise Exception('Input blob arguments do not match net inputs.')
# Set input according to defined shapes and make arrays single and
# C-contiguous as Caffe expects.
# in_为blob name, blob为 blob ndarray
for in_, blob in kwargs.iteritems():
if blob.shape[0] != self.blobs[in_].num:
raise Exception('Input is not batch sized')
self.blobs[in_].data[...] = blob
# 对应_caffe.cpp中的.def("_forward", &Net<Dtype>::ForwardFromTo),可以猜想应该是调用底层的Net<Dtype>::ForwardFromTo方法,进行前向传播
self._forward(start_ind, end_ind)
# Unpack blobs to extract
#rpn_test.pt所定义的网络的output为:rois blob 和 scores blob 两行log可以说明:
#I0429 03:42:10.559293 9520 net.cpp:270] This network produces output rois
#I0429 03:42:10.559300 9520 net.cpp:270] This network produces output scores
#outputs为以列表,其元素为网络所有输出blob的name
return {out: self.blobs[out].data for out in outputs}
- net.forwar() 调用的是Pycaffe.py中的_Net_forward函数,代码如上:(关于pycaffe.py —-> Wrap the internal caffe C++ module (_caffe.so) with a clean, Pythonic interface.)
- 在Pycaffe.py中,方法中带有self参数,个人觉得这应该表示一个Net对象,_Net_forward返回一个字典,{blob name: blob ndarray},
- start, end为可选的 layer name,注意是name
- kwargs : Keys are input *blob names and values are blob ndarrays.*
self._layer_names
:对应_caffe.cpp中的.add_property("_layer_names", bp::make_function(&Net<Dtype>::layer_names,bp::return_value_policy<bp::copy_const_reference>()))
,可以猜想应该是调用底层的Net::layer_names,返回网络中所有层的nameself._forward(start_ind, end_ind)
:对应_caffe.cpp中的.def("_forward", &Net<Dtype>::ForwardFromTo)
,可以猜想应该是调用底层的Net<Dtype>::ForwardFromTo
方法,进行前向传播outputs = set(self.outputs + blobs)
:对应pycaffe.py中@property def _Net_outputs(self): return [list(self.blobs.keys())[i] for i in self._outputs]
a) self._outputs(_caffe.cpp)调用底层的Net::output_blob_indices方法,返回网络所有输出blob的id ;
b) self.blobs.keys():
self.blobs对应pycaffe.py中的def _Net_blobs(self): """ An OrderedDict (bottom to top, i.e., input to output) of network blobs indexed by name """ return OrderedDict(zip(self._blob_names, self._blobs))
返回{blob name : blob ndarray} dict
所以self.outputs返回的应该是网络输出blob的name
总之, _caffe.cpp 和 pycaffe.py这两个文件要好好研究一下
def _get_image_blob(im)
def _get_image_blob(im):
"""Converts an image into a network input.也就是将图像转换为caffe所支持的输入数据结构即blob
Arguments:
im (ndarray): a color image in BGR order
Returns:
blob (ndarray): a data blob holding an image pyramid
im_scale_factors (list): list of image scales (relative to im) used
in the image pyramid
"""
im_orig = im.astype(np.float32, copy=True)
im_orig -= cfg.PIXEL_MEANS
im_shape = im_orig.shape
im_size_min = np.min(im_shape[0:2])
im_size_max = np.max(im_shape[0:2])
processed_ims = []
assert len(cfg.TEST.SCALES) == 1
target_size = cfg.TEST.SCALES[0]
im_scale = float(target_size) / float(im_size_min)
# Prevent the biggest axis from being more than MAX_SIZE
if np.round(im_scale * im_size_max) > cfg.TEST.MAX_SIZE:
im_scale = float(cfg.TEST.MAX_SIZE) / float(im_size_max)
im = cv2.resize(im_orig, None, None, fx=im_scale, fy=im_scale,
interpolation=cv2.INTER_LINEAR)
# im_info,一些图像信息:H W scale, 而且数据结构为[[H, W, scale]]
#np.newaxis添加了一个新轴,但是,新坐标轴上没有元素
im_info = np.hstack((im.shape[:2], im_scale))[np.newaxis, :]
processed_ims.append(im)
# Create a blob to hold the input images
#调用blob.py中的im_list_to_blob将图像转换为caffe所支持的数据结构blob,所做的工作就是复制数据,调整通道顺序,im_list_to_blob返回的其实是np.ndarray
blob = im_list_to_blob(processed_ims)
#返回的blob, im_info其实都是np.ndarray
return blob, im_info