比赛网址
https://god.yanxishe.com/18
train.py
import os, sys, glob, argparse
import pandas as pd
import numpy as np
from tqdm import tqdm
import time, datetime
import pdb, traceback
import cv2
from PIL import Image
from sklearn.model_selection import train_test_split, StratifiedKFold, KFold
from efficientnet_pytorch import EfficientNet
import torch
torch.manual_seed(0)
torch.backends.cudnn.deterministic = False
torch.backends.cudnn.benchmark = True
import torchvision.models as models
import torchvision.transforms as transforms
import torchvision.datasets as datasets
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.autograd import Variable
from torch.utils.data.dataset import Dataset
train_bboxes = pd.read_csv('./train_bboxes.csv')
train_label_neg = train_bboxes['filename'].value_counts().reset_index()
pos_img = [x.split('/')[-1][:-4] for x in glob.glob('./VOC2007/JPEGImages/*.jpg')]
pos_img = set(pos_img) - set(train_label_neg['index'])
pos_img = list(pos_img)
train_label_pos = pd.DataFrame({'index': pos_img, 'filename': [0]*len(pos_img)})
train_label = pd.concat([train_label_pos, train_label_neg], axis=0)
train_label = train_label.sample(frac=1)
train_label['index'] = train_label['index'].apply(lambda x: './VOC2007/JPEGImages/'+str(x)+'.jpg')
class QRDataset(Dataset):
def __init__(self, img_df, transform=None):
self.img_df = img_df
if transform is not None:
self.transform = transform
else:
self.transform = None
def __getitem__(self, index):
start_time = time.time()
img = Image.open(self.img_df.iloc[index]['index']).convert('RGB')
if self.transform is not None:
img = self.transform(img)
return img,torch.from_numpy(np.array(self.img_df.iloc[index]['filename']))
def __len__(self):
return len(self.img_df)
def accuracy(output, target, topk=(1,)):
"""Computes the accuracy over the k top predictions for the specified values of k"""
with torch.no_grad():
maxk = max(topk)
batch_size = target.size(0)
_, pred = output.topk(maxk, 1, True, True)
pred = pred.t()
correct = pred.eq(target.view(1, -1).expand_as(pred))
res = []
for k in topk:
correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)
res.append(correct_k.mul_(100.0 / batch_size))
return res
class AverageMeter(object):
"""Computes and stores the average and current value"""
def __init__(self, name, fmt=':f'):
self.name = name
self.fmt = fmt
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def __str__(self):
fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})'
return fmtstr.format(**self.__dict__)
class ProgressMeter(object):
def __init__(self, num_batches, *meters):
self.batch_fmtstr = self._get_batch_fmtstr(num_batches)
self.meters = meters
self.prefix = ""
def pr2int(self, batch):
entries = [self.prefix + self.batch_fmtstr.format(batch)]
entries += [str(meter) for meter in self.meters]
print('\t'.join(entries))
def _get_batch_fmtstr(self, num_batches):
num_digits = len(str(num_batches // 1))
fmt = '{:' + str(num_digits) + 'd}'
return '[' + fmt + '/' + fmt.format(num_batches) + ']'
class VisitNet(nn.Module):
def __init__(self):
super(VisitNet, self).__init__()
model = models.resnet18(True)
model.avgpool = nn.AdaptiveAvgPool2d(1)
model.fc = nn.Linear(512, 5)
self.resnet = model
def forward(self, img):
out = self.resnet(img)
return out
def validate(val_loader, model, criterion):
batch_time = AverageMeter('Time', ':6.3f')
losses = AverageMeter('Loss', ':.4e')
top1 = AverageMeter('Acc@1', ':6.2f')
top5 = AverageMeter('Acc@2', ':6.2f')
progress = ProgressMeter(len(val_loader), batch_time, losses, top1, top5)
model.eval()
with torch.no_grad():
end = time.time()
for i, (input, target) in enumerate(val_loader):
input = input.cuda()
target = target.cuda()
output = model(input)
loss = criterion(output, target)
acc1, acc5 = accuracy(output, target, topk=(1, 2))
losses.update(loss.item(), input.size(0))
top1.update(acc1[0], input.size(0))
top5.update(acc5[0], input.size(0))
batch_time.update(time.time() - end)
end = time.time()
print(' * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'
.format(top1=top1, top5=top5))
return top1
def predict(test_loader, model, tta=10):
model.eval()
test_pred_tta = None
for _ in range(tta):
test_pred = []
with torch.no_grad():
end = time.time()
for i, (input, target) in enumerate(test_loader):
input = input.cuda()
target = target.cuda()
output = model(input, path)
output = output.data.cpu().numpy()
test_pred.append(output)
test_pred = np.vstack(test_pred)
if test_pred_tta is None:
test_pred_tta = test_pred
else:
test_pred_tta += test_pred
return test_pred_tta
def train(train_loader, model, criterion, optimizer, epoch):
batch_time = AverageMeter('Time', ':6.3f')
losses = AverageMeter('Loss', ':.4e')
top1 = AverageMeter('Acc@1', ':6.2f')
progress = ProgressMeter(len(train_loader), batch_time, losses, top1)
model.train()
end = time.time()
for i, (input, target) in enumerate(train_loader):
input = input.cuda(non_blocking=True)
target = target.cuda(non_blocking=True)
output = model(input)
loss = criterion(output, target)
acc1, acc5 = accuracy(output, target, topk=(1, 2))
losses.update(loss.item(), input.size(0))
top1.update(acc1[0], input.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
if i % 100 == 0:
progress.pr2int(i)
skf = KFold(n_splits=10, random_state=233, shuffle=True)
for flod_idx, (train_idx, val_idx) in enumerate(skf.split(train_label['filename'].values, train_label['filename'].values)):
if flod_idx == 0:
continue
train_loader = torch.utils.data.DataLoader(
QRDataset(train_label.iloc[train_idx],
transforms.Compose([
transforms.Resize((512, 512)),
transforms.RandomHorizontalFlip(),
transforms.RandomVerticalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
), batch_size=10, shuffle=True, num_workers=20, pin_memory=True
)
val_loader = torch.utils.data.DataLoader(
QRDataset(train_label.iloc[val_idx],
transforms.Compose([
transforms.Resize((512, 512)),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
), batch_size=10, shuffle=False, num_workers=10, pin_memory=True
)
model = VisitNet().cuda()
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(model.parameters(), 0.01)
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=4, gamma=0.85)
best_acc = 0.0
for epoch in range(10):
scheduler.step()
print('Epoch: ', epoch)
train(train_loader, model, criterion, optimizer, epoch)
val_acc = validate(val_loader, model, criterion)
if val_acc.avg.item() > best_acc:
best_acc = val_acc.avg.item()
torch.save(model.state_dict(), './resnet18_fold{0}.pt'.format(flod_idx))
predict.py
import os, sys, glob, argparse
import pandas as pd
import numpy as np
from tqdm import tqdm
import time, datetime
import pdb, traceback
import cv2
from PIL import Image
from sklearn.model_selection import train_test_split, StratifiedKFold, KFold
from efficientnet_pytorch import EfficientNet
import torch
torch.manual_seed(0)
torch.backends.cudnn.deterministic = False
torch.backends.cudnn.benchmark = True
import torchvision.models as models
import torchvision.transforms as transforms
import torchvision.datasets as datasets
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.autograd import Variable
from torch.utils.data.dataset import Dataset
class QRDataset(Dataset):
def __init__(self, img_path, transform=None):
self.img_path = img_path
if transform is not None:
self.transform = transform
else:
self.transform = None
def __getitem__(self, index):
start_time = time.time()
img = Image.open(self.img_path[index]).convert('RGB')
if self.transform is not None:
img = self.transform(img)
return img,torch.from_numpy(np.array(int('PNEUMONIA' in self.img_path[index])))
def __len__(self):
return len(self.img_path)
class VisitNet(nn.Module):
def __init__(self):
super(VisitNet, self).__init__()
model = models.resnet18(True)
model.avgpool = nn.AdaptiveAvgPool2d(1)
model.fc = nn.Linear(512, 5)
self.resnet = model
def forward(self, img):
out = self.resnet(img)
return out
def predict(test_loader, model, tta=10):
model.eval()
test_pred_tta = None
for _ in range(tta):
test_pred = []
with torch.no_grad():
end = time.time()
for i, (input, target) in enumerate(test_loader):
input = input.cuda()
target = target.cuda()
output = model(input)
output = output.data.cpu().numpy()
test_pred.append(output)
test_pred = np.vstack(test_pred)
if test_pred_tta is None:
test_pred_tta = test_pred
else:
test_pred_tta += test_pred
return test_pred_tta
test_jpg = ['./test/{0}.jpg'.format(x) for x in range(0, 6671)]
test_jpg = np.array(test_jpg)
test_pred = None
for model_path in ['resnet18_fold0.pt', 'resnet18_fold0.pt']:
test_loader = torch.utils.data.DataLoader(
QRDataset(test_jpg,
transforms.Compose([
transforms.Resize((512, 512)),
transforms.RandomHorizontalFlip(),
transforms.RandomVerticalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
), batch_size=10, shuffle=False, num_workers=10, pin_memory=True
)
model = VisitNet().cuda()
model.load_state_dict(torch.load(model_path))
if test_pred is None:
test_pred = predict(test_loader, model, 2)
else:
test_pred += predict(test_loader, model, 2)
test_csv = pd.DataFrame()
test_csv[0] = list(range(0, 6671))
test_csv[1] = np.argmax(test_pred, 1)
test_csv.to_csv('tmp.csv', index=None, header=None)