数据增强方式：调整亮度，对比度，色调，随机缩放，剪切，翻转，旋转等；
Mosaic data augmentation：
- Mixup：两张图象按照不同的比例混合，分类结果按照混合比例分配；
- Cutout：随机将图片中的区域大小（个人观点裁剪区域图像大小为0.25左右较为合适，比例再大准确率下降明显）裁剪用0填充，结果分类不发生变化；
- Cutmix：随机裁剪图像的区域大小，不填充0而是随机选择其他样本的裁剪区域大小填充，结果根据裁剪填充的比例分类【Cutmix = Mixup + Cutout】。
对比：
Mixup和Cutmix：前者是将两张图根据设定比例插值融合，如图所示，若隐若现。
Cutout和Cutmix：前者对随机区域大小使用0填充，后者两张图像的填充。而yolo中类似cutmix使用4张图。
import torch
import torch.nn as nn
import numpy as np
# cutmix和mixup的实现代码
def rand_bbox(size, lam):
"""
随机生成裁剪框大小区域
输入参数：图像大小，和随机生成的参数lambda
"""
W = size[2]
H = size[3]
# 计算裁剪框区域大小的w和h；
cut_rat = np.sqrt(1. - lam)
cut_w = np.int(W * cut_rat)
cut_h = np.int(H * cut_rat)
# uniform
# randint函数计算中心点，此处计算中心点
cx = np.random.randint(W)
cy = np.random.randint(H)
# 裁剪区域大小不能超过W和H
bbx1 = np.clip(cx - cut_w // 2, 0, W)
bby1 = np.clip(cy - cut_h // 2, 0, H)
bbx2 = np.clip(cx + cut_w // 2, 0, W)
bby2 = np.clip(cy + cut_h // 2, 0, H)
# 返回四个点
return bbx1, bby1, bbx2, bby2
def cutmix(data, targets1, targets2, targets3, alpha):
indices = torch.randperm(data.size(0))
shuffled_data = data[indices]
shuffled_targets1 = targets1[indices]
shuffled_targets2 = targets2[indices]
shuffled_targets3 = targets3[indices]
lam = np.random.beta(alpha, alpha) # 随机生成lambda，符合beta分布
bbx1, bby1, bbx2, bby2 = rand_bbox(data.size(), lam) # 获得裁剪框
data[:, :, bbx1:bbx2, bby1:bby2] = data[indices, :, bbx1:bbx2, bby1:bby2]
# adjust lambda to exactly match pixel ratio
lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (data.size()[-1] * data.size()[-2]))
targets = [targets1, shuffled_targets1, targets2, shuffled_targets2, targets3, shuffled_targets3, lam]
return data, targets
def mixup(data, targets1, targets2, targets3, alpha):
indices = torch.randperm(data.size(0))
shuffled_data = data[indices]
shuffled_targets1 = targets1[indices]
shuffled_targets2 = targets2[indices]
shuffled_targets3 = targets3[indices]
lam = np.random.beta(alpha, alpha)
data = data * lam + shuffled_data * (1 - lam)
targets = [targets1, shuffled_targets1, targets2, shuffled_targets2, targets3, shuffled_targets3, lam]
return data, targets
def cutmix_criterion(preds1,preds2,preds3, targets):
targets1, targets2,targets3, targets4,targets5, targets6, lam = targets[0], targets[1], targets[2], targets[3], targets[4], targets[5], targets[6]
criterion = nn.CrossEntropyLoss(reduction='mean') # 交叉熵损失计算
return lam * criterion(preds1, targets1) + (1 - lam) * criterion(preds1, targets2) + lam * criterion(preds2, targets3) + (1 - lam) * criterion(preds2, targets4) + lam * criterion(preds3, targets5) + (1 - lam) * criterion(preds3, targets6)
def mixup_criterion(preds1,preds2,preds3, targets):
targets1, targets2,targets3, targets4,targets5, targets6, lam = targets[0], targets[1], targets[2], targets[3], targets[4], targets[5], targets[6]
criterion = nn.CrossEntropyLoss(reduction='mean')
return lam * criterion(preds1, targets1) + (1 - lam) * criterion(preds1, targets2) + lam * criterion(preds2, targets3) + (1 - lam) * criterion(preds2, targets4) + lam * criterion(preds3, targets5) + (1 - lam) * criterion(preds3, targets6)
# ==========================================================================================
for i, (image_id, images, label1, label2, label3) in enumerate(data_loader_train):
images = images.to(device)
label1 = label1.to(device)
label2 = label2.to(device)
label3 = label3.to(device)
# print (image_id, label1, label2, label3)
if np.random.rand()<0.5:
images, targets = mixup(images, label1, label2, label3, 0.4)
output1, output2, output3 = model(images)
loss = mixup_criterion(output1,output2,output3, targets)
else:
images, targets = cutmix(images, label1, label2, label3, 0.4)
output1, output2, output3 = model(images)
loss = cutmix_criterion(output1,output2,output3, targets)
=====================================================================
import glob
import numpy as np
import matplotlib.pyplot as plt
# import torch.nn
plt.rcParams['figure.figsize'] = [10, 10] # 裁剪框大小
import cv2
# 数据导入路劲
data_folder = f"./file/"
# 文件夹读取
filenames = glob.glob(f"{data_folder}*.png")
image_paths = filenames[:4] # 图片数量，
# 显示第一张图片
res = cv2.cvtColor(cv2.imread(image_paths[0]), cv2.COLOR_BGR2RGB)
plt.imshow(res)
plt.show()
image_batch = []
image_batch_labels = []
n_images = 4 # 设置文件里包含图片总数
for i in range(4):
image = cv2.cvtColor(cv2.imread(image_paths[i]), cv2.COLOR_BGR2RGB)
image_batch.append(image)
image_batch_labels = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]])
# for i in range(2):
# for j in range(2):
# plt.subplot(2,2,2*i+j+1)
# plt.imshow(image_batch[2*i+j])
# plt.show()
# 随机大小框和随机生成的lambda
def rand_bbox(size, lamb):
W = size[0]
H = size[1]
cut_rat = np.sqrt(1. - lamb)
cut_w = np.int(W * cut_rat)
cut_h = np.int(H * cut_rat)
# uniform
cx = np.random.randint(W)
cy = np.random.randint(H)
bbx1 = np.clip(cx - cut_w // 2, 0, W)
bby1 = np.clip(cy - cut_h // 2, 0, H)
bbx2 = np.clip(cx + cut_w // 2, 0, W)
bby2 = np.clip(cy + cut_h // 2, 0, H)
return bbx1, bby1, bbx2, bby2
# ============================== 显示rand_bbox效果 ==================================
image = cv2.cvtColor(cv2.imread(image_paths[0]), cv2.COLOR_BGR2RGB)
# 裁剪任意lamb大小
lamb = 0.3
size = image.shape
print('size', image.shape)
bbox = rand_bbox(size, lamb)
# 在图像上画框框
im = image.copy()
x1 = bbox[0]
y1 = bbox[1]
x2 = bbox[2]
y2 = bbox[3]
cv2.rectangle(im, (x1, y1), (x2, y2), (255, 0, 0), 3) # cv画矩形
plt.imshow(im)
plt.title('Original image with random bounding box')
plt.show()
# Show cropped image
plt.imshow(image[y1:y2, x1:x2]);
plt.title('Cropped image')
plt.show()
# ==================================================================================
def generate_cutmix_image(image_batch, image_batch_labels, beta):
# 生成混合sample
lam = np.random.beta(beta, beta)
rand_index = np.random.permutation(len(image_batch))
target_a = image_batch_labels
target_b = np.array(image_batch_labels)[rand_index]
print('img.shape', image_batch[0].shape)
bbx1, bby1, bbx2, bby2 = rand_bbox(image_batch[0].shape, lam)
print('bbx1', bbx1)
print('bby1', bby1)
print('bbx2', bbx2)
print('bby1', bby1)
image_batch_updated = image_batch.copy()
image_batch_updated = np.array(image_batch_updated)
image_batch = np.array(image_batch)
image_batch_updated[:, bbx1:bby1, bbx2:bby2, :] = image_batch[rand_index, bbx1:bby1, bbx2:bby2, :]
# 调整计算lambda精确像素比例
lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (image_batch.shape[1] * image_batch.shape[2]))
label = target_a * lam + target_b * (1. - lam)
return image_batch_updated, label
# image_batch=np.array(image_batch)
# image_batch_updated = image_batch.copy()
# c=[1,0,2,3]
# mm=np.array(image_batch_updated)
# mm[:, 10:200, 10:200, :] = image_batch[c, 10:200, 10:200, :]
# Generate CutMix image
# Let's use the first image of the batch as the input image to be augmented
input_image = image_batch[0]
image_batch_updated, image_batch_labels_updated = generate_cutmix_image(image_batch, image_batch_labels, 1.0)
# Show original images
print("Original Images")
for i in range(2):
for j in range(2):
plt.subplot(2, 2, 2 * i + j + 1)
plt.imshow(image_batch[2 * i + j])
plt.show()
# Show CutMix images
print("CutMix Images")
for i in range(2):
for j in range(2):
plt.subplot(2, 2, 2 * i + j + 1)
plt.imshow(image_batch_updated[2 * i + j])
plt.show()
# Print labels
print('Original labels:')
print(image_batch_labels)
print('Updated labels')
print(image_batch_labels_updated)
=====================================================================
Mosaic登场：Yolov4的mosaic数据增强在CutMix数据增强基础上，利用了四张图片进行拼接得到，其优点是丰富检测物体的背景，且减少计算每个BN中的数据4倍。
读取四张图片->->->->对四张图片进行翻转、缩放、色域变化等，并且按照四个方向位置摆好->->->->图片的组合和框的组合
代码实现
from PIL import Image, ImageDraw
import numpy as np
from matplotlib.colors import rgb_to_hsv, hsv_to_rgb
import math
def rand(a=0, b=1):
return np.random.rand()*(b-a) + a
def merge_bboxes(bboxes, cutx, cuty):
merge_bbox = []
for i in range(len(bboxes)):
for box in bboxes[i]:
tmp_box = []
x1,y1,x2,y2 = box[0], box[1], box[2], box[3]
if i == 0:
if y1 > cuty or x1 > cutx:
continue
if y2 >= cuty and y1 <= cuty:
y2 = cuty
if y2-y1 < 5:
continue
if x2 >= cutx and x1 <= cutx:
x2 = cutx
if x2-x1 < 5:
continue
if i == 1:
if y2 < cuty or x1 > cutx:
continue
if y2 >= cuty and y1 <= cuty:
y1 = cuty
if y2-y1 < 5:
continue
if x2 >= cutx and x1 <= cutx:
x2 = cutx
if x2-x1 < 5:
continue
if i == 2:
if y2 < cuty or x2 < cutx:
continue
if y2 >= cuty and y1 <= cuty:
y1 = cuty
if y2-y1 < 5:
continue
if x2 >= cutx and x1 <= cutx:
x1 = cutx
if x2-x1 < 5:
continue
if i == 3:
if y1 > cuty or x2 < cutx:
continue
if y2 >= cuty and y1 <= cuty:
y2 = cuty
if y2-y1 < 5:
continue
if x2 >= cutx and x1 <= cutx:
x1 = cutx
if x2-x1 < 5:
continue
tmp_box.append(x1)
tmp_box.append(y1)
tmp_box.append(x2)
tmp_box.append(y2)
tmp_box.append(box[-1])
merge_bbox.append(tmp_box)
return merge_bbox
def get_random_data(annotation_line, input_shape, random=True, hue=.1, sat=1.5, val=1.5, proc_img=True):
'''random preprocessing for real-time data augmentation'''
h, w = input_shape
min_offset_x = 0.4
min_offset_y = 0.4
scale_low = 1-min(min_offset_x,min_offset_y)
scale_high = scale_low+0.2
image_datas = []
box_datas = []
index = 0
place_x = [0,0,int(w*min_offset_x),int(w*min_offset_x)]
place_y = [0,int(h*min_offset_y),int(w*min_offset_y),0]
for line in annotation_line:
# 每一行进行分割
line_content = line.split()
# 打开图片
image = Image.open(line_content[0])
image = image.convert("RGB")
# 图片的大小
iw, ih = image.size
# 保存框的位置
box = np.array([np.array(list(map(int,box.split(',')))) for box in line_content[1:]])
# image.save(str(index)+".jpg")
# 是否翻转图片
flip = rand()<.5
if flip and len(box)>0:
image = image.transpose(Image.FLIP_LEFT_RIGHT)
box[:, [0,2]] = iw - box[:, [2,0]]
# 对输入进来的图片进行缩放
new_ar = w/h
scale = rand(scale_low, scale_high)
if new_ar < 1:
nh = int(scale*h)
nw = int(nh*new_ar)
else:
nw = int(scale*w)
nh = int(nw/new_ar)
image = image.resize((nw,nh), Image.BICUBIC)
# 进行色域变换
hue = rand(-hue, hue)
sat = rand(1, sat) if rand()<.5 else 1/rand(1, sat)
val = rand(1, val) if rand()<.5 else 1/rand(1, val)
x = rgb_to_hsv(np.array(image)/255.)
x[..., 0] += hue
x[..., 0][x[..., 0]>1] -= 1
x[..., 0][x[..., 0]<0] += 1
x[..., 1] *= sat
x[..., 2] *= val
x[x>1] = 1
x[x<0] = 0
image = hsv_to_rgb(x)
image = Image.fromarray((image*255).astype(np.uint8))
# 将图片进行放置，分别对应四张分割图片的位置
dx = place_x[index]
dy = place_y[index]
new_image = Image.new('RGB', (w,h), (128,128,128))
new_image.paste(image, (dx, dy))
image_data = np.array(new_image)/255
# Image.fromarray((image_data*255).astype(np.uint8)).save(str(index)+"distort.jpg")
index = index + 1
box_data = []
# 对box进行重新处理
if len(box)>0:
np.random.shuffle(box)
box[:, [0,2]] = box[:, [0,2]]*nw/iw + dx
box[:, [1,3]] = box[:, [1,3]]*nh/ih + dy
box[:, 0:2][box[:, 0:2]<0] = 0
box[:, 2][box[:, 2]>w] = w
box[:, 3][box[:, 3]>h] = h
box_w = box[:, 2] - box[:, 0]
box_h = box[:, 3] - box[:, 1]
box = box[np.logical_and(box_w>1, box_h>1)]
box_data = np.zeros((len(box),5))
box_data[:len(box)] = box
image_datas.append(image_data)
box_datas.append(box_data)
img = Image.fromarray((image_data*255).astype(np.uint8))
for j in range(len(box_data)):
thickness = 3
left, top, right, bottom = box_data[j][0:4]
draw = ImageDraw.Draw(img)
for i in range(thickness):
draw.rectangle([left + i, top + i, right - i, bottom - i],outline=(255,255,255))
img.show()
# 将图片分割，放在一起
cutx = np.random.randint(int(w*min_offset_x), int(w*(1 - min_offset_x)))
cuty = np.random.randint(int(h*min_offset_y), int(h*(1 - min_offset_y)))
new_image = np.zeros([h,w,3])
new_image[:cuty, :cutx, :] = image_datas[0][:cuty, :cutx, :]
new_image[cuty:, :cutx, :] = image_datas[1][cuty:, :cutx, :]
new_image[cuty:, cutx:, :] = image_datas[2][cuty:, cutx:, :]
new_image[:cuty, cutx:, :] = image_datas[3][:cuty, cutx:, :]
# 对框进行进一步的处理
new_boxes = merge_bboxes(box_datas, cutx, cuty)
return new_image, new_boxes
if __name__ == "__main__":
with open("2007_train.txt") as f:
lines = f.readlines()
a = np.random.randint(0,len(lines))
line = lines[a:a+4]
image_data, box_data = get_random_data(line,[416,416])
img = Image.fromarray((image_data*255).astype(np.uint8))
for j in range(len(box_data)):
thickness = 3
left, top, right, bottom = box_data[j][0:4]
draw = ImageDraw.Draw(img)
for i in range(thickness):
draw.rectangle([left + i, top + i, right - i, bottom - i],outline=(255,255,255))
img.show()
# img.save("box_all.jpg")
1345
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
