百度架构师手把手带你零基础实践深度学习——YOLO-V3

发展历程

2013年，Ross Girshick 等人于首次将CNN的方法应用在目标检测任务上，他们使用传统图像算法Selective Search产生候选区域，取得了极大的成功，这就是对目标检测领域影响深远的区域卷积神经网络(R-CNN)模型。
2015年，Ross Girshick 对此方法进行了改进，提出了Fast R-CNN模型。通过将不同区域的物体共用卷积层的计算，大大缩减了计算量，提高了处理速度，而且还引入了调整目标物体位置的回归方法，进一步提高了位置预测的准确性。
2015年，Shaoqing Ren 等人提出了Faster R-CNN模型，提出了RPN的方法来产生物体的候选区域，这一方法不再需要使用传统的图像处理算法来产生候选区域，进一步提升了处理速度。
2017年，Kaiming He 等人提出了Mask R-CNN模型，只需要在Faster R-CNN模型上添加比较少的计算量，就可以同时实现目标检测和物体实例分割两个任务。
以上都是基于R-CNN系列的著名模型，对目标检测方向的发展有着较大的影响力。此外，还有一些其他模型，比如SSD、YOLO(1, 2, 3)、R-FCN等也都是目标检测领域流行的模型结构。

目标检测基础概念

边界框（bounding box）
锚框（Anchor box)
交并比（IoU）

数据集及预处理

size：图片尺寸

object：图片中包含的物体，一张图片可能中包含多个物体

name：昆虫名称
bndbox：物体真实框
difficult：识别是否困难

数据读取

import cv2

def get_bbox(gt_bbox, gt_class):
# 对于一般的检测任务来说，一张图片上往往会有多个目标物体
# 设置参数MAX_NUM = 50， 即一张图片最多取50个真实框；如果真实
# 框的数目少于50个，则将不足部分的gt_bbox, gt_class和gt_score的各项数值全设置为0
MAX_NUM = 50
gt_bbox2 = np.zeros((MAX_NUM, 4))
gt_class2 = np.zeros((MAX_NUM,))
for i in range(len(gt_bbox)):
gt_bbox2[i, :] = gt_bbox[i, :]
gt_class2[i] = gt_class[i]
if i >= MAX_NUM:
break
return gt_bbox2, gt_class2

def get_img_data_from_file(record):
“”"
record is a dict as following,
record = {
‘im_file’: img_file,
‘im_id’: im_id,
‘h’: im_h,
‘w’: im_w,
‘is_crowd’: is_crowd,
‘gt_class’: gt_class,
‘gt_bbox’: gt_bbox,
‘gt_poly’: [],
‘difficult’: difficult
}
“”"
im_file = record[‘im_file’]
h = record[‘h’]
w = record[‘w’]
is_crowd = record[‘is_crowd’]
gt_class = record[‘gt_class’]
gt_bbox = record[‘gt_bbox’]
difficult = record[‘difficult’]

img = cv2.imread(im_file)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

# check if h and w in record equals that read from img
assert img.shape[0] == int(h), \
         "image height of {} inconsistent in record({}) and img file({})".format(
           im_file, h, img.shape[0])

assert img.shape[1] == int(w), \
         "image width of {} inconsistent in record({}) and img file({})".format(
           im_file, w, img.shape[1])

gt_boxes, gt_labels = get_bbox(gt_bbox, gt_class)

# gt_bbox 用相对值
gt_boxes[:, 0] = gt_boxes[:, 0] / float(w)
gt_boxes[:, 1] = gt_boxes[:, 1] / float(h)
gt_boxes[:, 2] = gt_boxes[:, 2] / float(w)
gt_boxes[:, 3] = gt_boxes[:, 3] / float(h)

return img, gt_boxes, gt_labels, (h, w)

数据预处理

图像增广方法汇总：
随机改变亮暗、对比度和颜色等
随机填充
随机裁剪
随机缩放
随机翻转
随机打乱真实框排列顺序

图像增广主要作用：扩大训练数据集，抑制过拟合，提升模型的泛化能力，常用的方法见下面的程序。

import numpy as np
import cv2
from PIL import Image, ImageEnhance
import random

图像增广方法汇总

def image_augment(img, gtboxes, gtlabels, size, means=None):
# 随机改变亮暗、对比度和颜色等
img = random_distort(img)
# 随机填充
img, gtboxes = random_expand(img, gtboxes, fill=means)
# 随机裁剪
img, gtboxes, gtlabels, = random_crop(img, gtboxes, gtlabels)
# 随机缩放
img = random_interp(img, size)
# 随机翻转
img, gtboxes = random_flip(img, gtboxes)
# 随机打乱真实框排列顺序
gtboxes, gtlabels = shuffle_gtbox(gtboxes, gtlabels)

return img.astype('float32'), gtboxes.astype('float32'), gtlabels.astype('int32')

#随机改变亮暗、对比度和颜色等
def random_distort(img):
# 随机改变亮度
def random_brightness(img, lower=0.5, upper=1.5):
e = np.random.uniform(lower, upper)
return ImageEnhance.Brightness(img).enhance(e)
# 随机改变对比度
def random_contrast(img, lower=0.5, upper=1.5):
e = np.random.uniform(lower, upper)
return ImageEnhance.Contrast(img).enhance(e)
# 随机改变颜色
def random_color(img, lower=0.5, upper=1.5):
e = np.random.uniform(lower, upper)
return ImageEnhance.Color(img).enhance(e)

ops = [random_brightness, random_contrast, random_color]
np.random.shuffle(ops)

img = Image.fromarray(img)
img = ops[0](img)
img = ops[1](img)
img = ops[2](img)
img = np.asarray(img)

return img

#随机填充
def random_expand(img,
gtboxes,
max_ratio=4.,
fill=None,
keep_ratio=True,
thresh=0.5):
if random.random() > thresh:
return img, gtboxes

if max_ratio < 1.0:
    return img, gtboxes

h, w, c = img.shape
ratio_x = random.uniform(1, max_ratio)
if keep_ratio:
    ratio_y = ratio_x
else:
    ratio_y = random.uniform(1, max_ratio)
oh = int(h * ratio_y)
ow = int(w * ratio_x)
off_x = random.randint(0, ow - w)
off_y = random.randint(0, oh - h)

out_img = np.zeros((oh, ow, c))
if fill and len(fill) == c:
    for i in range(c):
        out_img[:, :, i] = fill[i] * 255.0

out_img[off_y:off_y + h, off_x:off_x + w, :] = img
gtboxes[:, 0] = ((gtboxes[:, 0] * w) + off_x) / float(ow)
gtboxes[:, 1] = ((gtboxes[:, 1] * h) + off_y) / float(oh)
gtboxes[:, 2] = gtboxes[:, 2] / ratio_x
gtboxes[:, 3] = gtboxes[:, 3] / ratio_y

return out_img.astype('uint8'), gtboxes

import numpy as np

def multi_box_iou_xywh(box1, box2):
“”"
In this case, box1 or box2 can contain multi boxes.
Only two cases can be processed in this method:
1, box1 and box2 have the same shape, box1.shape == box2.shape
2, either box1 or box2 contains only one box, len(box1) == 1 or len(box2) == 1
If the shape of box1 and box2 does not match, and both of them contain multi boxes, it will be wrong.
“”"
assert box1.shape[-1] == 4, “Box1 shape[-1] should be 4.”
assert box2.shape[-1] == 4, “Box2 shape[-1] should be 4.”

b1_x1, b1_x2 = box1[:, 0] - box1[:, 2] / 2, box1[:, 0] + box1[:, 2] / 2
b1_y1, b1_y2 = box1[:, 1] - box1[:, 3] / 2, box1[:, 1] + box1[:, 3] / 2
b2_x1, b2_x2 = box2[:, 0] - box2[:, 2] / 2, box2[:, 0] + box2[:, 2] / 2
b2_y1, b2_y2 = box2[:, 1] - box2[:, 3] / 2, box2[:, 1] + box2[:, 3] / 2

inter_x1 = np.maximum(b1_x1, b2_x1)
inter_x2 = np.minimum(b1_x2, b2_x2)
inter_y1 = np.maximum(b1_y1, b2_y1)
inter_y2 = np.minimum(b1_y2, b2_y2)
inter_w = inter_x2 - inter_x1
inter_h = inter_y2 - inter_y1
inter_w = np.clip(inter_w, a_min=0., a_max=None)
inter_h = np.clip(inter_h, a_min=0., a_max=None)

inter_area = inter_w * inter_h
b1_area = (b1_x2 - b1_x1) * (b1_y2 - b1_y1)
b2_area = (b2_x2 - b2_x1) * (b2_y2 - b2_y1)

return inter_area / (b1_area + b2_area - inter_area)

def box_crop(boxes, labels, crop, img_shape):
x, y, w, h = map(float, crop)
im_w, im_h = map(float, img_shape)

boxes = boxes.copy()
boxes[:, 0], boxes[:, 2] = (boxes[:, 0] - boxes[:, 2] / 2) * im_w, (
    boxes[:, 0] + boxes[:, 2] / 2) * im_w
boxes[:, 1], boxes[:, 3] = (boxes[:, 1] - boxes[:, 3] / 2) * im_h, (
    boxes[:, 1] + boxes[:, 3] / 2) * im_h

crop_box = np.array([x, y, x + w, y + h])
centers = (boxes[:, :2] + boxes[:, 2:]) / 2.0
mask = np.logical_and(crop_box[:2] <= centers, centers <= crop_box[2:]).all(
    axis=1)

boxes[:, :2] = np.maximum(boxes[:, :2], crop_box[:2])
boxes[:, 2:] = np.minimum(boxes[:, 2:], crop_box[2:])
boxes[:, :2] -= crop_box[:2]
boxes[:, 2:] -= crop_box[:2]

mask = np.logical_and(mask, (boxes[:, :2] < boxes[:, 2:]).all(axis=1))
boxes = boxes * np.expand_dims(mask.astype('float32'), axis=1)
labels = labels * mask.astype('float32')
boxes[:, 0], boxes[:, 2] = (boxes[:, 0] + boxes[:, 2]) / 2 / w, (
    boxes[:, 2] - boxes[:, 0]) / w
boxes[:, 1], boxes[:, 3] = (boxes[:, 1] + boxes[:, 3]) / 2 / h, (
    boxes[:, 3] - boxes[:, 1]) / h

return boxes, labels, mask.sum()

#随机裁剪
def random_crop(img,
boxes,
labels,
scales=[0.3, 1.0],
max_ratio=2.0,
constraints=None,
max_trial=50):
if len(boxes) == 0:
return img, boxes

if not constraints:
    constraints = [(0.1, 1.0), (0.3, 1.0), (0.5, 1.0), (0.7, 1.0),
                   (0.9, 1.0), (0.0, 1.0)]

img = Image.fromarray(img)
w, h = img.size
crops = [(0, 0, w, h)]
for min_iou, max_iou in constraints:
    for _ in range(max_trial):
        scale = random.uniform(scales[0], scales[1])
        aspect_ratio = random.uniform(max(1 / max_ratio, scale * scale), \
                                      min(max_ratio, 1 / scale / scale))
        crop_h = int(h * scale / np.sqrt(aspect_ratio))
        crop_w = int(w * scale * np.sqrt(aspect_ratio))
        crop_x = random.randrange(w - crop_w)
        crop_y = random.randrange(h - crop_h)
        crop_box = np.array([[(crop_x + crop_w / 2.0) / w,
                              (crop_y + crop_h / 2.0) / h,
                              crop_w / float(w), crop_h / float(h)]])

        iou = multi_box_iou_xywh(crop_box, boxes)
        if min_iou <= iou.min() and max_iou >= iou.max():
            crops.append((crop_x, crop_y, crop_w, crop_h))
            break

while crops:
    crop = crops.pop(np.random.randint(0, len(crops)))
    crop_boxes, crop_labels, box_num = box_crop(boxes, labels, crop, (w, h))
    if box_num < 1:
        continue
    img = img.crop((crop[0], crop[1], crop[0] + crop[2],
                    crop[1] + crop[3])).resize(img.size, Image.LANCZOS)
    img = np.asarray(img)
    return img, crop_boxes, crop_labels
img = np.asarray(img)
return img, boxes, labels

#随机缩放
def random_interp(img, size, interp=None):
interp_method = [
cv2.INTER_NEAREST,
cv2.INTER_LINEAR,
cv2.INTER_AREA,
cv2.INTER_CUBIC,
cv2.INTER_LANCZOS4,
]
if not interp or interp not in interp_method:
interp = interp_method[random.randint(0, len(interp_method) - 1)]
h, w, _ = img.shape
im_scale_x = size / float(w)
im_scale_y = size / float(h)
img = cv2.resize(
img, None, None, fx=im_scale_x, fy=im_scale_y, interpolation=interp)
return img

#随机翻转
def random_flip(img, gtboxes, thresh=0.5):
if random.random() > thresh:
img = img[:, ::-1, :]
gtboxes[:, 0] = 1.0 - gtboxes[:, 0]
return img, gtboxes

#随机打乱真实框排列顺序
def shuffle_gtbox(gtbox, gtlabel):
gt = np.concatenate(
[gtbox, gtlabel[:, np.newaxis]], axis=1)
idx = np.arange(gt.shape[0])
np.random.shuffle(idx)
gt = gt[idx, :]
return gt[:, :4], gt[:, 4]

批量数据读取与加速

通过使用飞桨提供的API paddle.reader.xmap_readers可以开启多线程读取数据.
import functools
import paddle

使用paddle.reader.xmap_readers实现多线程读取数据

def multithread_loader(datadir, batch_size= 10, mode=‘train’):
cname2cid = get_insect_names()
records = get_annotations(cname2cid, datadir)
def reader():
if mode == ‘train’:
np.random.shuffle(records)
img_size = get_img_size(mode)
batch_data = []
for record in records:
batch_data.append((record, img_size))
if len(batch_data) == batch_size:
yield batch_data
batch_data = []
img_size = get_img_size(mode)
if len(batch_data) > 0:
yield batch_data

def get_data(samples):
    batch_data = []
    for sample in samples:
        record = sample[0]
        img_size = sample[1]
        img, gt_bbox, gt_labels, im_shape = get_img_data(record, size=img_size)
        batch_data.append((img, gt_bbox, gt_labels, im_shape))
    return make_array(batch_data)

mapper = functools.partial(get_data, )

return paddle.reader.xmap_readers(mapper, reader, 8, 10)

YOLO-V3模型设计思想

1. 按一定规则在图片上产生一系列的候选区域，然后根据这些候选区域与图片上物体真实框之间的位置关系对候选区域进行标注。跟真实框足够接近的那些候选区域会被标注为正样本，同时将真实框的位置作为正样本的位置目标。偏离真实框较大的那些候选区域则会被标注为负样本，负样本不需要预测位置或者类别。

2. 使用卷积神经网络提取图片特征并对候选区域的位置和类别进行预测。这样每个预测框就可以看成是一个样本，根据真实框相对它的位置和类别进行了标注而获得标签值，通过网络模型预测其位置和类别，将网络预测值和标签值进行比较，就可以建立起损失函数。

产生候选区域

生成锚框
生成预测框
对候选区域进行标注

标注锚框的具体程序
#标注预测框的objectness
def get_objectness_label(img, gt_boxes, gt_labels, iou_threshold = 0.7,
anchors = [116, 90, 156, 198, 373, 326],
num_classes=7, downsample=32):
“”"
img 是输入的图像数据，形状是[N, C, H, W]
gt_boxes，真实框，维度是[N, 50, 4]，其中50是真实框数目的上限，当图片中真实框不足50个时，不足部分的坐标全为0
真实框坐标格式是xywh，这里使用相对值
gt_labels，真实框所属类别，维度是[N, 50]
iou_threshold，当预测框与真实框的iou大于iou_threshold时不将其看作是负样本
anchors，锚框可选的尺寸
anchor_masks，通过与anchors一起确定本层级的特征图应该选用多大尺寸的锚框
num_classes，类别数目
downsample，特征图相对于输入网络的图片尺寸变化的比例
“”"

img_shape = img.shape
batchsize = img_shape[0]
num_anchors = len(anchors) // 2
input_h = img_shape[2]
input_w = img_shape[3]
# 将输入图片划分成num_rows x num_cols个小方块区域，每个小方块的边长是 downsample
# 计算一共有多少行小方块
num_rows = input_h // downsample
# 计算一共有多少列小方块
num_cols = input_w // downsample

label_objectness = np.zeros([batchsize, num_anchors, num_rows, num_cols])
label_classification = np.zeros([batchsize, num_anchors, num_classes, num_rows, num_cols])
label_location = np.zeros([batchsize, num_anchors, 4, num_rows, num_cols])

scale_location = np.ones([batchsize, num_anchors, num_rows, num_cols])

# 对batchsize进行循环，依次处理每张图片
for n in range(batchsize):
    # 对图片上的真实框进行循环，依次找出跟真实框形状最匹配的锚框
    for n_gt in range(len(gt_boxes[n])):
        gt = gt_boxes[n][n_gt]
        gt_cls = gt_labels[n][n_gt]
        gt_center_x = gt[0]
        gt_center_y = gt[1]
        gt_width = gt[2]
        gt_height = gt[3]
        if (gt_height < 1e-3) or (gt_height < 1e-3):
            continue
        i = int(gt_center_y * num_rows)
        j = int(gt_center_x * num_cols)
        ious = []
        for ka in range(num_anchors):
            bbox1 = [0., 0., float(gt_width), float(gt_height)]
            anchor_w = anchors[ka * 2]
            anchor_h = anchors[ka * 2 + 1]
            bbox2 = [0., 0., anchor_w/float(input_w), anchor_h/float(input_h)]
            # 计算iou
            iou = box_iou_xywh(bbox1, bbox2)
            ious.append(iou)
        ious = np.array(ious)
        inds = np.argsort(ious)
        k = inds[-1]
        label_objectness[n, k, i, j] = 1
        c = gt_cls
        label_classification[n, k, c, i, j] = 1.

        # for those prediction bbox with objectness =1, set label of location
        dx_label = gt_center_x * num_cols - j
        dy_label = gt_center_y * num_rows - i
        dw_label = np.log(gt_width * input_w / anchors[k*2])
        dh_label = np.log(gt_height * input_h / anchors[k*2 + 1])
        label_location[n, k, 0, i, j] = dx_label
        label_location[n, k, 1, i, j] = dy_label
        label_location[n, k, 2, i, j] = dw_label
        label_location[n, k, 3, i, j] = dh_label
        # scale_location用来调节不同尺寸的锚框对损失函数的贡献，作为加权系数和位置损失函数相乘
        scale_location[n, k, i, j] = 2.0 - gt_width * gt_height

# 目前根据每张图片上所有出现过的gt box，都标注出了objectness为正的预测框，剩下的预测框则默认objectness为0
# 对于objectness为1的预测框，标出了他们所包含的物体类别，以及位置回归的目标
return label_objectness.astype('float32'), label_location.astype('float32'), label_classification.astype('float32'), \
         scale_location.astype('float32')

卷积神经网络提取特征

YOLO-V3算法使用的骨干网络是Darknet53。

import paddle.fluid as fluid
from paddle.fluid.param_attr import ParamAttr
from paddle.fluid.regularizer import L2Decay

from paddle.fluid.dygraph.nn import Conv2D, BatchNorm
from paddle.fluid.dygraph.base import to_variable

YOLO-V3骨干网络结构Darknet53的实现代码

class ConvBNLayer(fluid.dygraph.Layer):
“”"
卷积 + 批归一化，BN层之后激活函数默认用leaky_relu
“”"
def init(self,
ch_in,
ch_out,
filter_size=3,
stride=1,
groups=1,
padding=0,
act=“leaky”,
is_test=True):
super(ConvBNLayer, self).init()

    self.conv = Conv2D(
        num_channels=ch_in,
        num_filters=ch_out,
        filter_size=filter_size,
        stride=stride,
        padding=padding,
        groups=gr