关于yolo v1的理论解释,网上已经有很多很详细的讲解,这里就不在赘述,只是网上对loss的解释也仅仅限于以下公式层面的说明,但是代码解释较少,这里将详细讲述基于Python的代码实现(仅说明loss的代码部分)
原始数据处理
假设原始数据集是voc2007,文件夹VOC2007\JPEGImages存放图片集,VOC2007\Annotations存放标注文件(xml格式),具体组织结构,这里不再赘述,其他数据集文件结构可能不同,但是处理后的数据格式都是下列要求
-
图片处理
根据yolo论文,输入网络的图片大小为 448 ∗ 448 448\ast448 448∗448,因此,调整原始图片大小最为输入。def image_read(self, imname, flipped=False): image = cv2.imread(imname) image = cv2.resize(image, (self.image_size, self.image_size)) # 调整图像大小到448*448 self.image_size=448 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32) image = (image / 255.0) * 2.0 - 1.0 # 使图像像素值介于-1~1之间 if flipped: # 数据增强,是否翻转图像 image = image[:, ::-1, :] return image
-
标注文件处理
yolo论文中,输入的gt(ground_truth)格式是(center_x, center_y, gt_w, gt_h, class_id),即物体真实边框的中心点坐标+真实边框高和宽+物体类别。另外,yolo是将 448 ∗ 448 448\ast448 448∗448的图像按 7 ∗ 7 7\ast7 7∗7的格网进行划分(并非真的划分,而是经过卷积网络输出的特征图大小是 7 ∗ 7 7\ast7 7∗7),所以对于一批输入图像,对应label的形状就是[batch_size, 7, 7, 25],(25=1(标记是否有物体)+4(gt_box)+20(类别one_hot编码)如果物体中心点在某一单元格内,就记录gt,没有则是0,如果多个物体中心在同一个单元格内,取其中一个物体的gt(这也是yolo v1的缺陷之一)。计算物体中心点所在单元格公式:
x _ i n d = ⌊ c e n t e r _ x ∗ 7 / 448 ⌋ x\_ind=\lfloor center\_x\ast7/448 \rfloor x_ind=⌊center_x∗7/448⌋
y _ i n d = ⌊ c e n t e r _ y ∗ 7 / 448 ⌋ y\_ind=\lfloor center\_y \ast7/448 \rfloor y_ind=⌊center_y∗7/448⌋
其实就是计算 448 ∗ 448 448*448 448∗448图像下的坐标对应在 7 ∗ 7 7*7 7∗7特征图下的坐标。def load_pascal_annotation(self, index): """ 处理单张图片标注文件 self.image_size: 448 self.cell_size: 7 """ imname = os.path.join(self.data_path, 'JPEGImages', index + '.jpg') im = cv2.imread(imname) h_ratio = 1.0 * self.image_size / im.shape[0] w_ratio = 1.0 * self.image_size / im.shape[1] # im = cv2.resize(im, [self.image_size, self.image_size]) label = np.zeros((self.cell_size, self.cell_size, 25)) # 初始化label,形状[7, 7, 25] filename = os.path.join(self.data_path, 'Annotations', index + '.xml') tree = ET.parse(filename) objs = tree.findall('object') for obj in objs: bbox = obj.find('bndbox') # Make pixel indexes 0-based x1 = max(min((float(bbox.find('xmin').text) - 1) * w_ratio, self.image_size - 1), 0) y1 = max(min((float(bbox.find('ymin').text) - 1) * h_ratio, self.image_size - 1), 0) x2 = max(min((float(bbox.find('xmax').text) - 1) * w_ratio, self.image_size - 1), 0) y2 = max(min((float(bbox.find('ymax').text) - 1) * h_ratio, self.image_size - 1), 0) cls_ind = self.class_to_ind[obj.find('name').text.lower().strip()] boxes = [(x2 + x1) / 2.0, (y2 + y1) / 2.0, x2 - x1, y2 - y1] # gt_box # 计算中心点所在单元格索引 x_ind = int(boxes[0] * self.cell_size / self.image_size) y_ind = int(boxes[1] * self.cell_size / self.image_size) if label[y_ind, x_ind, 0] == 1: # 当前单元格是否有物体 continue label[y_ind, x_ind, 0] = 1 label[y_ind, x_ind, 1:5] = boxes label[y_ind, x_ind, 5 + cls_ind] = 1 return label, len(objs) def load_labels(self): """ 处理所有标注数据 """ if self.phase == 'train': txtname = os.path.join( self.data_path, 'ImageSets', 'Main', 'trainval.txt') else: txtname = os.path.join( self.data_path, 'ImageSets', 'Main', 'test.txt') with open(txtname, 'r') as f: self.image_index = [x.strip() for x in f.readlines()] gt_labels = [] for index in self.image_index: label, num = self.load_pascal_annotation(index) if num == 0: continue imname = os.path.join(self.data_path, 'JPEGImages', index + '.jpg') gt_labels.append({'imname': imname, 'label': label, 'flipped': False}) return gt_labels
loss计算
输入图像进入网络结构的到的输出预测结果形状是[batch_size, 7, 7, 30],其中30=2*(4+1)+20,即一个单元格预测两个边界框,每个边界框是[pre_center_y, pre_center_x, p r e _ w \sqrt {pre\_w} pre_w, p r e _ h \sqrt {pre\_h} pre_h] (相对于所在单元格左上角)+confidence(置信度),所以是2*(4+1),虽然一个单元格预测两个边界框,但是一个单元格只预测一次物体类别,共20类(yolo v1只能预测20类物体,这也是缺陷之一)
真实置信度,根据yolo论文,是根据 P r ( o b j ) ∗ I O U P_r(obj)*IOU Pr(obj)∗IOU计算的,而 P r ( o b j ) P_r(obj) Pr(obj)的值只能是0或1,所以真实置信度要么是IOU,要么是0直接上代码,代码注释即为解释
def loss_layer(self, predicts, labels, scope='loss_layer'): """ self.boundary1: 20 物体类别数量 self.boundary2: 2 置信度个数 """ with tf.variable_scope(scope): predict_classes = tf.reshape(predicts[:, :self.boundary1], [self.batch_size, self.cell_size, self.cell_size, self.num_class]) # 预测类别 predict_scales = tf.reshape(predicts[:, self.boundary1:self.boundary2], [self.batch_size, self.cell_size, self.cell_size, self.boxes_per_cell]) # 预测置信度 predict_boxes = tf.reshape(predicts[:, self.boundary2:], [self.batch_size, self.cell_size, self.cell_size, self.boxes_per_cell, 4]) # 预测边框 response = tf.reshape( labels[..., 0], [self.batch_size, self.cell_size, self.cell_size, 1]) # 在7*7的格网里每个单元格是否有物体 boxes = tf.reshape(labels[..., 1:5], [self.batch_size, self.cell_size, self.cell_size, 1, 4]) boxes = tf.tile(boxes, [1, 1, 1, self.boxes_per_cell, 1]) / self.image_size # 归一化 classes = labels[..., 5:] offset = tf.reshape(tf.constant(self.offset, dtype=tf.float32), [1, self.cell_size, self.cell_size, self.boxes_per_cell]) offset = tf.tile(offset, [self.batch_size, 1, 1, 1]) offset_tran = tf.transpose(offset, (0, 2, 1, 3)) predict_boxes_tran = tf.stack( [(predict_boxes[..., 0] + offset) / self.cell_size, # 计算预测框center_x在特征图中的位置 (predict_boxes[..., 1] + offset_tran) / self.cell_size, # 计算预测框center_y在特征图中的位置 predict_boxes[..., 2], predict_boxes[..., 3]], axis=-1) iou_predict_truth = self.calc_iou(predict_boxes_tran, boxes) # 计算交并比 # calculate I tensor [BATCH_SIZE, CELL_SIZE, CELL_SIZE, BOXES_PER_CELL] object_mask = tf.reduce_max(iou_predict_truth, 3, keep_dims=True) # 计算每个单元格的2个预测框中IOU中最大值 object_mask = tf.cast( (iou_predict_truth >= object_mask), tf.float32) * response # 单元格中有最大IOU物体的设置为1,其余为0 # calculate no_I tensor [CELL_SIZE, CELL_SIZE, BOXES_PER_CELL] noobject_mask = tf.ones_like( object_mask, dtype=tf.float32) - object_mask # 与object_mask 相反 boxes_tran = tf.stack( [boxes[..., 0] * self.cell_size - offset, # 之前已经计算boxes/image_size, 这里相当于x*cell_size/image_size,即计算真实位置在特征图上的位置 boxes[..., 1] * self.cell_size - offset_tran, # 同上 tf.sqrt(boxes[..., 2]), # 对w开方 tf.sqrt(boxes[..., 3])], # 对h开方 axis=-1) # class_loss class_delta = response * (predict_classes - classes) class_loss = tf.reduce_mean( tf.reduce_sum(tf.square(class_delta), axis=[1, 2, 3]), name='class_loss') * self.class_scale # object_loss object_delta = object_mask * (predict_scales - iou_predict_truth) # 有物体,真实置信度是IOU object_loss = tf.reduce_mean( tf.reduce_sum(tf.square(object_delta), axis=[1, 2, 3]), name='object_loss') * self.object_scale # noobject_loss noobject_delta = noobject_mask * (predict_scales - 0) # 没物体,真实置信度是0 noobject_loss = tf.reduce_mean( tf.reduce_sum(tf.square(noobject_delta), axis=[1, 2, 3]), name='noobject_loss') * self.noobject_scale # coord_loss coord_mask = tf.expand_dims(object_mask, 4) boxes_delta = coord_mask * (predict_boxes - boxes_tran) coord_loss = tf.reduce_mean( tf.reduce_sum(tf.square(boxes_delta), axis=[1, 2, 3, 4]), name='coord_loss') * self.coord_scale tf.losses.add_loss(class_loss) tf.losses.add_loss(object_loss) tf.losses.add_loss(noobject_loss) tf.losses.add_loss(coord_loss) tf.summary.scalar('class_loss', class_loss) tf.summary.scalar('object_loss', object_loss) tf.summary.scalar('noobject_loss', noobject_loss) tf.summary.scalar('coord_loss', coord_loss) tf.summary.histogram('boxes_delta_x', boxes_delta[..., 0]) tf.summary.histogram('boxes_delta_y', boxes_delta[..., 1]) tf.summary.histogram('boxes_delta_w', boxes_delta[..., 2]) tf.summary.histogram('boxes_delta_h', boxes_delta[..., 3]) tf.summary.histogram('iou', iou_predict_truth)
有的时候,原理懂了,转换为代码还是需要一些方法的。
以上解释仅为个人理解,有不足或错误的地方,望大牛们批评指正。