文章目录
1.比赛简介
1.1 链接
https://www.kaggle.com/c/seti-breakthrough-listen
1.2 简介
“宇宙中只有我们吗?”
这是人类最深刻和永恒的问题之一。随着技术的进步,我们正在寻找新的、更强大的方法来寻找答案。加利福尼亚大学的突破侦探小组(伯克利)利用世界上最强大的望远镜来扫描数百万颗恒星,寻找技术的迹象。现在它希望Kaggle社区帮助解释他们接收到的信号。
Listen团队是搜寻外星智能(SETI)的一部分,使用地球上最大的可操纵碟形望远镜,直径100米的格林班克望远镜。与任何SETI搜索一样,沟通的动机也是主要的挑战。人类已经制造了大量的无线电设备。在现代科技探测的大海捞针中,很难找到外星传播的微弱信号。
当前的方法使用两个过滤器在草堆中搜索。首先,Listen团队将目标恒星的扫描与天空其他区域的扫描穿插在一起。两组扫描中出现的任何信号都可能不是来自目标恒星的方向。其次,管道会丢弃不改变频率的信号,因为这意味着它们可能在望远镜附近。运动中的震源应该有一个指示移动的信号,类似于经过的消防车警报器的音调变化。这两个过滤器相当有效,但我们知道它们可以改进。毫无疑问,管道漏掉了感兴趣的信号,特别是那些具有复杂时间或频率结构的信号,以及那些在频谱中存在大量干扰的区域的信号。
在本次比赛中,使用您的数据科学技能帮助识别突破性目标扫描中的异常信号。由于目前还没有确定的外星信号用于训练机器学习算法的例子,研究小组在望远镜的大量数据中加入了一些模拟信号(他们称之为“针”)。他们已经确定了一些隐藏的针,这样你就可以训练你的模型来发现更多。数据由二维阵列组成,因此可能有计算机视觉的方法很有前途,还有数字信号处理、异常检测等。成功识别最多针头的算法将获得现金奖励,但也有可能帮助回答科学中最大的问题之一。
2.数据下载
链接: https://pan.baidu.com/s/1wr8m9ZOo7PpafZYKHjrw5g 提取码: 4wdj
3.baseline代码
3.1 EfficientNet pytorch
import os
import sys
sys.path = ['../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master',] + sys.path
import pandas as pd
import glob
import numpy as np
import matplotlib.pyplot as plt
import cv2
from sklearn.model_selection import train_test_split
from sklearn import metrics
from tqdm import tqdm
import torch
import torchvision.models as models
import torch.nn as nn
from efficientnet_pytorch import model as enet
import random
from sklearn.model_selection import KFold, StratifiedKFold
import albumentations as A
import torch.optim as optim
import torch.nn.functional as F
"""
设置显卡的使用数量
"""
os.environ['CUDA_VISIBLE_DEVICES'] = '0,1,2,3'
if torch.cuda.device_count() > 1:
print("We can use", torch.cuda.device_count(), "GPUs!")
"""
设置是否并行
"""
parallel=False
"""
选择使用gpu或者cpu运行
"""
if torch.cuda.is_available():
device = torch.device("cuda")
print("GPU is available")
else:
device = torch.device("cpu")
print("GPU not available, CPU used")
"""
设置随即种子,达到可以复现的效果
"""
def set_seed(seed = 0):
'''Sets the seed of the entire notebook so results are the same every time we run.
This is for REPRODUCIBILITY.'''
np.random.seed(seed)
random_state = np.random.RandomState(seed)
random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
os.environ['PYTHONHASHSEED'] = str(seed)
return random_state
seed = 666
random_state = set_seed(seed)
"""
对图像进行增强,在dataload的时候进行使用
"""
# img_sz = 256
img_sz = 512
ttransform = A.Compose([
A.Resize(img_sz, img_sz, cv2.INTER_NEAREST),
#A.VerticalFlip(p=0.5),
#A.HorizontalFlip(p=0.5),
])
vtransform = A.Compose([
A.Resize(img_sz, img_sz, cv2.INTER_NEAREST),
#A.VerticalFlip(p=0.5),
#A.HorizontalFlip(p=0.5),
])
class ClassificationDataset:
"""
加载数据dataloader方法
"""
def __init__(self, image_paths, targets, tr):
"""
初始化对图像进行处理
"""
self.image_paths = image_paths
self.targets = targets
self.tr = tr
def __len__(self):
return len(self.image_paths)
def __getitem__(self, item):
"""
将图像转为tensor
"""
image = np.load(self.image_paths[item])
image = np.vstack(image).astype(float)
image = self.tr(image = image)["image"][np.newaxis, ]
targets = self.targets[item]
return {
"image": torch.tensor(image, dtype=torch.float),
"targets": torch.tensor(targets, dtype=torch.long),
}
class FocalLoss(nn.Module):
"""
定义focalloss,这里主要针对二分类标签不均衡问题
"""
def __init__(self, alpha=1, gamma=2):
super(FocalLoss, self).__init__()
self.alpha = alpha
self.gamma = gamma
self.epsilon = 1e-12 # prevent training from Nan-loss error
def forward(self, logits, target):
"""
logits & target should be tensors with shape [batch_size, num_classes]
"""
probs = F.sigmoid(logits)
one_subtract_probs = 1.0 - probs
# add epsilon
probs_new = probs + self.epsilon
one_subtract_probs_new = one_subtract_probs + self.epsilon
# calculate focal loss
log_pt = target * torch.log(probs_new) + (1.0 - target) * torch.log(one_subtract_probs_new)
pt = torch.exp(log_pt)
focal_loss = -1.0 * (self.alpha * (1 - pt) ** self.gamma) * log_pt
return torch.mean(focal_loss)
class enetv2(nn.Module):
"""
主干网络
"""
def __init__(self, backbone, out_dim):
"""
:param backbone: 主干网络
:param out_dim: 输出的维度,这里只有一个线性输出,故输出维度为1
"""
super(enetv2, self).__init__()
self.enet = enet.EfficientNet.from_name(backbone)
self.enet.load_state_dict(torch.load(pretrained_model[backbone]))#默认会加载官方的预训练模型参数
self.myfc = nn.Linear(self.enet._fc.in_features, out_dim)
self.enet._fc = nn.Identity()
self.conv1 = nn.Conv2d(1, 3, kernel_size=3, stride=1, padding=3, bias=False)
def extract(self, x):
return self.enet(x)
def forward(self, x):
x = self.conv1(x)
x = self.extract(x)
x = self.myfc(x)
return x
def mixup_data(x, y, alpha=1.0, use_cuda=True):
"""
混合精读提速
:param x:
:param y:
:param alpha:
:param use_cuda:
:return:
"""
'''Returns mixed inputs, pairs of targets, and lambda'''
if alpha > 0:
lam = np.random.beta(alpha, alpha)
else:
lam = 1
batch_size = x.size()[0]
if use_cuda:
index = torch.randperm(batch_size).cuda()
else:
index = torch.randperm(batch_size)
mixed_x = lam * x + (1 - lam) * x[index, :]
y_a, y_b = y, y[index]
return mixed_x, y_a, y_b, lam
def mixup_criterion(criterion, pred, y_a, y_b, lam):
"""
:param criterion: 损失函数
:param pred:
:param y_a:
:param y_b:
:param lam:
:return:
"""
return lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b)
def train(data_loader, model, optimizer, device):
"""
训练数据函数
:param data_loader:
:param model:
:param optimizer:
:param device:
:return:
"""
criterion = FocalLoss()
model.train()
for data in tqdm(data_loader, position=0, leave=True, desc='Training'):
inputs1 = data["image"]
targets = data['targets']
inputs1, targets_a, targets_b, lam = mixup_data(inputs1, targets.view(-1, 1), use_cuda=True)
inputs1 = inputs1.to(device, dtype=torch.float)
targets_a = targets_a.to(device, dtype=torch.float)
targets_b = targets_b.to(device, dtype=torch.float)
optimizer.zero_grad()
outputs = model(inputs1)
loss = mixup_criterion(criterion, outputs, targets_a, targets_b, lam)
loss.backward()
optimizer.step()
def evaluate(data_loader, model, device):
"""
数据推理
:param data_loader:
:param model:
:param device:
:return:
"""
model.eval()
final_targets = []
final_outputs = []
with torch.no_grad():
for data in tqdm(data_loader, position=0, leave=True, desc='Evaluating'):
inputs = data["image"]
targets = data["targets"]
inputs = inputs.to(device, dtype=torch.float)
targets = targets.to(device, dtype=torch.float)
output = model(inputs)
targets = targets.detach().cpu().numpy().tolist()
output = output.detach().cpu().numpy().tolist()
final_targets.extend(targets)
final_outputs.extend(output)
return final_outputs, final_targets
"""
设置官方预训练模型
"""
paths = [
'efficientnet-b0-08094119.pth',
'efficientnet-b1-dbc7070a.pth',
'efficientnet-b2-27687264.pth',
'efficientnet-b3-c8376fa2.pth',
'efficientnet-b4-e116e8b3.pth',
'efficientnet-b5-586e6cc6.pth',
'efficientnet-b6-c76e70fd.pth',
'efficientnet-b7-dcc49843.pth',
]
"""设置一些超参数"""
use_mode_index="4"
baseline_name = 'efficientnet-b{}'.format(use_mode_index)
pretrained_model = {
'efficientnet-b{}'.format(use_mode_index): '../input/efficientnet-pytorch/' + paths[int(use_mode_index)]
}
models = []
device = "cuda"
# df = df.sample(n = 1000).reset_index(drop=True)
epochs = 10
Batch_Size = 32
skf = StratifiedKFold(n_splits=5)
fold = 0
# criterion = nn.BCEWithLogitsLoss()
best_roc_auc=0
"""读取数据"""
df = pd.read_csv('../input/seti-breakthrough-listen/train_labels.csv')
print (df.shape)
df['img_path'] = df['id'].apply(lambda x: f'../input/seti-breakthrough-listen/train/{x[0]}/{x}.npy')
# df = pd.read_csv('../input/seti-breakthrough-listen/old_leaky_data/train_labels_old.csv')
# print (df.shape)
# df['img_path'] = df['id'].apply(lambda x: f'../input/seti-breakthrough-listen/old_leaky_data/train_old/{x[0]}/{x}.npy')
X = df.img_path.values
Y = df.target.values
for train_index, test_index in skf.split(X, Y):
model = enetv2(baseline_name, out_dim=1)
if parallel == True:
print("Let's use", torch.cuda.device_count(), "GPUs!")
model = nn.DataParallel(model)
model.to(device)
model.module.load_state_dict(torch.load("efficientnet-b4_0.pt"))
else:
model.to(device)
model.load_state_dict(torch.load("efficientnet-b4_0.pt"))
train_images, valid_images = X[train_index], X[test_index]
train_targets, valid_targets = Y[train_index], Y[test_index]
train_dataset = ClassificationDataset(image_paths=train_images, targets=train_targets, tr=ttransform)
valid_dataset = ClassificationDataset(image_paths=valid_images, targets=valid_targets, tr=vtransform)
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=Batch_Size,shuffle=True, num_workers=4)
valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=Batch_Size,shuffle=False, num_workers=4)
optimizer = torch.optim.AdamW(model.parameters(), lr=5e-4)
# optimizer = optim.AdamW(model.parameters(), weight_decay=1e-02)
# Scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=1e-3,epochs=20, pct_start=0.1,
# anneal_strategy='cos',div_factor=1e+3, final_div_factor=1e+3, steps_per_epoch=len(train_loader))
for epoch in range(epochs):
train(train_loader, model, optimizer, device=device)
predictions, valid_targets = evaluate(valid_loader, model, device=device)
roc_auc = metrics.roc_auc_score(valid_targets, predictions)
print(f"Epoch={epoch}, Valid ROC AUC={roc_auc}")
if roc_auc > best_roc_auc:
if parallel == True:
torch.save(model.module.state_dict(), baseline_name + '_new_data' + str(fold) + '.pt')
else:
torch.save(model.state_dict(), baseline_name + '_new_data' + str(fold) + '.pt')
best_roc_auc = roc_auc
fold += 1
"""
结果推理
"""
submission = pd.read_csv('../input/seti-breakthrough-listen/sample_submission.csv')
submission['img_path'] = submission['id'].apply(lambda x: f'../input/seti-breakthrough-listen/test/{x[0]}/{x}.npy')
"""推理时将数据进行预处理,即tta Test Time Augmentation"""
test_dataset = ClassificationDataset(image_paths=submission.img_path.values, targets=submission.target.values, tr=ttransform)
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=16, shuffle=False, num_workers=4)
test_predictions, test_targets = evaluate(test_loader, model, device=device)
"""保存推理结果"""
test_predictions = np.array(test_predictions)
submission.target = test_predictions[:, 0]
submission.drop(['img_path'], axis=1, inplace=True)
submission.to_csv('submission_B4_new_1v1.csv', index=False)
3.2 EfficientNet keras
import numpy as np
import pandas as pd
from pathlib import Path
import math
import tensorflow as tf
from tensorflow import keras
from sklearn import model_selection
import efficientnet.tfkeras as efn
"""
数据读取以及参数设置
"""
data_dir = Path('../input/seti-breakthrough-listen/')
train_data_dir = data_dir / 'train'
test_data_dir = data_dir / 'test'
train_label_file = data_dir / 'train_labels.csv'
sample_file = data_dir / 'sample_submission.csv'
label = pd.read_csv(train_label_file, index_col='id')
sub = pd.read_csv(sample_file, index_col='id')
def id_to_path(s, train=True):
data_dir = train_data_dir if train else test_data_dir
return data_dir / s[0] / f'{s}.npy'
input_size = (273, 256, 3)
batch_size = 32
n_epoch = 2
seed = 42
"""
定义数据加载
"""
class DataGenerator(keras.utils.Sequence):
def __init__(self, x_set, y_set=None, batch_size=32):
self.x , self.y = x_set, y_set
self.batch_size = batch_size
self.is_train = False if y_set is None else True
def __len__(self):
return math.ceil(len(self.x) / self.batch_size)
def __getitem__(self, idx):
batch_ids = self.x[idx * self.batch_size: (idx + 1) * self.batch_size]
if self.y is not None:
batch_y = self.y[idx * self.batch_size: (idx + 1) * self.batch_size]
list_x = [np.load(id_to_path(x, self.is_train))[::2] for x in batch_ids]
batch_x = np.moveaxis(list_x,1,-1)
batch_x = batch_x.astype("float") / 255
if self.is_train:
return batch_x, batch_y
else:
return batch_x
"""
定义模型
"""
model = tf.keras.Sequential([
efn.EfficientNetB3(input_shape=input_size,weights='imagenet',include_top=False),
keras.layers.GlobalAveragePooling2D(),
keras.layers.Dense(1, activation='sigmoid')
])
model.summary()
model.compile(optimizer=keras.optimizers.Adam(learning_rate=1e-4),
loss='binary_crossentropy', metrics=[keras.metrics.AUC()])
"""
训练模型
"""
x0 = label.index.values
y0 = label['target'].values
x1 = sub.index.values
x_train, x_val, y_train, y_val = model_selection.train_test_split(x0, y0, test_size=.2, random_state=seed)
train = DataGenerator(x_train, y_train, batch_size=batch_size)
val = DataGenerator(x_val, y_val, batch_size=batch_size)
test = DataGenerator(x1, batch_size=batch_size)
model.fit(train, validation_data=val, epochs=n_epoch)
"""
模型推理与结果生成
"""
prediction = model.predict(test).flatten()
sub['target'] = prediction
sub.to_csv('submission.csv')
3.3 Resnet18 pytorch
'''Load librarires'''
import pickle
import time
import random
import glob
import os
from copy import deepcopy
import numpy as np
import pandas as pd
from PIL import Image
from tqdm import tqdm
from pathlib import Path
from collections import defaultdict
import matplotlib.pyplot as plt
import plotly.express as px
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import Dataset,DataLoader
import torchvision
from torchvision import models, transforms, utils
import albumentations as A
from albumentations.pytorch import ToTensorV2
from albumentations import ImageOnlyTransform
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score, precision_score, recall_score, roc_curve, roc_auc_score
class CFG:
'''Store all hyperparameters here.'''
# DEBUG=False
SEED = 420
# TEST_SIZE = 0.2
VAL_SIZE = 0.1
CLASSES = None #Need to update manually
OUTPUT_FEATURES = None #Need to update manually
#Transforms
TRAIN_TRANSFORMS = A.Compose([
A.Resize(224, 224),
ToTensorV2(),
])
VAL_TRANSFORMS = A.Compose([
A.Resize(224, 224),
ToTensorV2(),
])
TEST_TRANSFORMS = A.Compose([
A.Resize(224, 224),
ToTensorV2(),
])
#model
MODEL1 = {
'name': 'resnet18',
'transfer': True,
'architecture': models.resnet18(pretrained=True), # ResNet18
'criterion': nn.CrossEntropyLoss(),
'optimizer': optim.Adam,
'weight_decay': 1e-6,
'lr': 1e-4,
'history': None
}
BATCH_SIZE = 192
EPOCHS = 3
DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print('You are using ->', DEVICE)
def seed_everything(seed):
'''Make the results reproducible'''
random.seed(seed)
os.environ['PYTHONHASHSEED'] = str(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(CFG.SEED)
train = pd.read_csv('../input/seti-breakthrough-listen/train_labels.csv')
sub = pd.read_csv('../input/seti-breakthrough-listen/sample_submission.csv')
'''Store data paths and their labels in pandas dataframe. Will be used to create pytorch datasets. '''
train_paths = glob.glob('../input/seti-breakthrough-listen/train/*/*' )
meta = pd.DataFrame(sorted(train_paths),columns=['path'])
#assign id and target to from the train df
meta['id'], meta['target'] = train.id, train.target
# get class mappings
classes = dict(enumerate(meta.target.astype('category').cat.categories))
CFG.CLASSES = classes
CFG.OUTPUT_FEATURES = len(CFG.CLASSES)
meta.head()
#get test data paths
test_paths = glob.glob('../input/seti-breakthrough-listen/test/*/*' )
test = pd.DataFrame(sorted(test_paths),columns=['path'])
test['target'] = 0 #dummy targets
test.head()
'''Split data into train, validation and test sets'''
X = list(meta.path)
y = list(meta.target)
X_train, X_val, y_train, y_val = train_test_split(X,
y,
test_size=CFG.VAL_SIZE,
random_state=CFG.SEED,
stratify=y) #stratified split
X_test,y_test = test.path, test.target
print(f'Train length -> {len(X_train)}')
print(f'Val length -> {len(X_val)}')
print(f'Test length -> {len(X_test)}')
'''Custom pytorch dataset implementation.'''
class SETIDataset(Dataset):
def __init__(self,X,y, transform=None):
self.X = X
self.y = y
self.transform = transform
assert len(self.X) == len(self.y), f'X and y have different lengths -> {len(self.X)} != {len(self.y)} '
def __len__(self):
return len(self.X)
def __getitem__(self,idx):
img_path = self.X[idx]
img = np.load(img_path)
img = img.astype(np.float32)
img = np.vstack(img).transpose((1, 0))
if self.transform is not None:
img = self.transform(image=img)['image']
label = torch.tensor(self.y[idx],dtype=torch.long)
return (img, label)
def show_img(self,idx):
'''Plot image'''
img,label = self.__getitem__(idx)
img = img.numpy().transpose((1, 2, 0))
plt.figure(figsize=(16, 8))
plt.axis('off')
plt.imshow(img)
plt.title(CFG.CLASSES[int(label)]) #using CFG.CLASSES dict
plt.pause(0.001)
'''Instantiate pytorch train, validation and test sets'''
TRAIN = SETIDataset(X_train,y_train, CFG.TRAIN_TRANSFORMS)
VAL = SETIDataset(X_val,y_val, CFG.VAL_TRANSFORMS)
TEST = SETIDataset(X_test,y_test, CFG.TEST_TRANSFORMS)
'''Instantiate Dataloaders'''
TRAIN_LOADER = DataLoader(TRAIN,CFG.BATCH_SIZE)
VAL_LOADER = DataLoader(VAL,CFG.BATCH_SIZE)
TEST_LOADER = DataLoader(TEST,CFG.BATCH_SIZE)
class Net(nn.Module):
'''
========================
NEURAL NET
========================
Args:
model_dict(dict): configuration dict containing the model architecture
output_features(int): length of output tensor; for classification equals to number of classes
'''
def __init__(self, model_dict, output_features):
super().__init__()
self.__dict__.update(model_dict) #unpack model dict from CFG into this class
if self.transfer:
model = self.architecture
#modify the last layer
num_ftrs = model.fc.in_features
model.fc = nn.Linear(num_ftrs, output_features)
#modify the input size to fit the grayscale images!
model.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3,
bias=False)
self.model = model
else:
# self.model = self.architecture
pass
self.output_features = output_features
#optimizer
self.optimizer = self.optimizer(self.model.parameters(),lr = self.lr, weight_decay = self.weight_decay,amsgrad=False)
#path where to save model
self.save_path = 'models'
def forward(self, x):
return self.model(x)
def fit(self,
train_loader,
val_loader,
epochs = 5,
batch_size = 32,
device = 'cpu'):
'''
=============================
OPTIMIZATION LOOP
=============================
Args:
train_loader(torch dataloader)
val_loader(torch dataloader)
epochs(int)
batch_size(int)
device(str)
Output style inspired by skorch fit() method
https://skorch.readthedocs.io/en/stable/net.html?highlight=fit#skorch.net.NeuralNet.fit
'''
#may be changed if lrscheduler is used???
lr = deepcopy(self.lr)
#get model training history
history = self.history
if history == None:
history = defaultdict(list)
else:
pass
#get train and val sizes
train_size = len(train_loader.dataset)
val_size = len(val_loader.dataset)
#stuff for printing epoch metrics as a beautiful table
headers = ['epoch','train_loss','val_loss','val_acc','cp','lr','dur']
template = '{:<10} {:<10} {:<10} {:<10} {:<10} {:<10} {:<10}'
print(template.format(*headers))
print(template.replace(':', ':-').format('','','','','','',''))
cyan = "\033[96m{:<10}\033[00m" #cyan
purple = "\033[95m{:<10}\033[00m" #purple
green = "\033[92m{:<10}\033[00m" #green
white = "\033[0m{:<10}\033[0m" #white
#set model into train mode
self.model.train()
#send model to device
self.model.to(device)
#training loop
for epoch in range(epochs):
start_time = time.time()
train_loss = 0
val_loss, val_acc = 0, 0
#optimization loop
time.sleep(.2)
for X,y in tqdm(train_loader, desc ="Train batches"):
#Send training data to device
X,y = X.to(device), y.to(device)
#Forward propagation
pred = self.model(X)
loss = self.criterion(pred,y)
#Backpropagation
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
#update loss
train_loss = loss.item()
#validation loop
with torch.no_grad():
for X, y in tqdm(val_loader, desc='Validation Batches'):
X,y = X.to(device),y.to(device)
pred = self.model(X)
val_loss = self.criterion(pred,y).item()
val_acc += (pred.argmax(1) == y).type(torch.float).sum().item()
#calculate validation accuracy after the epoch
val_acc /= val_size
#append epoch results
history['epoch'].append(epoch+1)
history['train_loss'].append(train_loss)
history['val_loss'].append(val_loss)
history['val_acc'].append(val_acc)
#colorize epoch's output if it improves
colortemp = template.split(' ')
# colorize train loss if it decreases
if history['train_loss'][-1] == min(history['train_loss']):
colortemp[1] = cyan
else:
colortemp[1] = white
#colorize validation loss if it decreases
if history['val_loss'][-1] == min(history['val_loss']):
colortemp[2] = purple
else:
colortemp[2] = white
# colorize validation accuracy & save best weights if it increases
if history['val_acc'][-1] == max(history['val_acc']):
#colorize
colortemp[3] = green
#checkpoint
cp = '+'
if not os.path.exists(self.save_path):
os.mkdir(self.save_path)
torch.save(self.model.state_dict(), Path(self.save_path,f'best_{self.name}.pth'))
else:
colortemp[3] = white
cp = '-'
colortemp = ' '.join(colortemp)
#calculate epoch duration (in seconds)
end_time = time.time()
dur = end_time - start_time
#append the rest of epoch results
history['cp'].append(cp)
history['lr'].append(lr)
history['dur'].append(dur)
#display the epoch results
print(colortemp.format(*f'{epoch+1}/{epochs} {train_loss:.4f} {val_loss:.4f} {val_acc:.2f} {cp} {lr} {dur:.2f}'.split(' ')))
#update epoch number of the entire training history
history['epoch'] = [e+1 for e in range(len(history['epoch']))]
#update model's training history
self.history = history
#save training history as csv
self.save_history()
def predict(self,dataloader,device ='cpu'):
'''
===============
Predict
===============
'''
#set model to evaluation mode
self.model.eval()
#model to device, default cpu
self.model.to(device)
preds = []
with torch.no_grad():
for X, y in tqdm(dataloader):
X,y = X.to(device),y.to(device)
pred = self.model(X)
pred = pred.argmax(1)
preds.append(pred)
return preds
def eval_model(self,dataloader,avg=None,device ='cpu'):
'''
==================================
ACCURACY PRECISION RECALL F1
==================================
'''
labels = [l for l in range(self.output_features)]
loader_size = len(dataloader)
dataset_size = len(dataloader.dataset)
acc = 0
precision = 0
recall = 0
f1 = 0
roc = 0
#set model to evaluation mode
self.model.eval()
#model to device, default cpu
self.model.to(device)
with torch.no_grad():
for X, y in tqdm(dataloader, desc = 'Evaluating the model'):
X,y = X.to(device),y.to(device)
pred = self.model(X)
#accuracy
acc += (pred.argmax(1) == y).type(torch.float).sum().item()
pred = pred.argmax(1)
# to int
pred,y = list(pred), list(y)
pred = [int(p) for p in pred]
y = [int(p) for p in y]
#precision
p = precision_score(y, pred, labels = labels, zero_division = 1, average = avg)
precision+=p
#recall
r = recall_score(y, pred, labels = labels, zero_division = 1, average = avg)
recall+=r
#f1 score
f = f1_score(y, pred, labels = labels, zero_division = 1, average = avg)
f1 += f
#roc
roc += roc_auc_score(y, pred)
acc /= dataset_size
precision /= loader_size
recall /= loader_size
f1 /= loader_size
roc /= loader_size
print(f" Accuracy: {(100*acc):>0.1f}%")
print(f"Precision: {(100*np.mean(precision)):>0.1f}%")
print(f" Recall: {(100*np.mean(recall)):>0.1f}%")
print(f" F1 Score: {(100*np.mean(f1)):>0.1f}%")
print(f" ROC: {(100*np.mean(f1)):>0.1f}%")
def plot_loss_history(self):
'''
Plot loss history
'''
assert self.history != None, 'No history to plot -> the model has not been trained yet!'
df = pd.DataFrame(self.history)
fig = px.line(x = df.epoch,
y = [df.train_loss, df.val_loss],
title = 'Loss History',
labels={'x':'epoch','value': 'loss', 'variable': 'loss'})
fig.data[0].name = 'train'
fig.data[1].name = 'val'
fig.show()
def save_history(self):
'''Save model's training history'''
assert self.history != None, 'No history to save -> the model has not been trained yet!'
#save as csv
pd.DataFrame(self.history).to_csv(Path(f'models/{self.name}_history.csv'))
#save as pickle file
with open(Path(f'models/{self.name}_history.pkl'), 'wb') as f:
pickle.dump(self.history, f, protocol=pickle.HIGHEST_PROTOCOL)
def save_model(self):
torch.save(self.model.state_dict(), Path(self.save_path,f'latest_{self.name}.pth'))
def load_model(self,path = 'models', device = 'cpu'):
'''Load model'''
try:
#load model weights
p = Path(path,f'best_{self.name}.pth')
self.model.load_state_dict(torch.load(p, map_location=torch.device(device)))
#load model training history
with open(Path(path,f'{self.name}_history.pkl'), 'rb') as h:
self.history = pickle.load(h)
except:
print('No model to load!')
"""
开始训练
"""
#instantiate model and send to device
Resnet18 = Net(CFG.MODEL1,CFG.OUTPUT_FEATURES)
Resnet18.load_model()
Resnet18.fit(TRAIN_LOADER,
VAL_LOADER,
CFG.EPOCHS,
CFG.BATCH_SIZE,
CFG.DEVICE)
#画出训练的loss图像
Resnet18.plot_loss_history()
#推理
#evaluate best model's performance on the VAL set在验证集上验证模型效果
print('Best Model:')
print('-'*20)
time.sleep(0.1)
best_model = deepcopy(Resnet18)
best_model.load_model(device = CFG.DEVICE)
best_model.eval_model(VAL_LOADER, avg = 'binary',device = CFG.DEVICE)
#evaluate current model's performance on the VAL set
print()
print('Current Model:')
time.sleep(0.1)
print('-'*20)
Resnet18.eval_model(VAL_LOADER,avg = 'binary', device = CFG.DEVICE)
'''
Get predictions on the TEST set在测试集上直接进行结果的推理并保存结果
'''
TEST_PREDS = Resnet18.predict(TEST_LOADER, CFG.DEVICE)
preds = [int(i) for p in TEST_PREDS for i in list(p)]
sub.target = preds
sub.to_csv('submission.csv', index=False)
sub.head()
3.3 GRU pytorch
# Libraries
import warnings
warnings.filterwarnings('ignore')
import numpy as np
import pandas as pd
import os
import glob
import matplotlib.pyplot as plt
%matplotlib inline
import seaborn as sns
import cv2
from tqdm import tqdm
from plotly.offline import iplot
from plotly.subplots import make_subplots
import plotly.express as px
import plotly.graph_objects as go
import plotly.figure_factory as ff
import tensorflow as tf
from tensorflow.keras import layers
import torch
import torchvision.models as models
import torch.nn as nn
from tensorflow.keras.optimizers import Adam
from tensorflow.keras import models
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau
from skimage.io import imshow, imread, imsave
from skimage.transform import rotate, AffineTransform, warp,rescale, resize, downscale_local_mean
from skimage import color,data
from skimage.exposure import adjust_gamma
from skimage.util import random_noise
"""
数据预处理
"""
data_dir = '../input/seti-breakthrough-listen'
train_merger = os.path.join(data_dir,'train_labels.csv')
train_labels = pd.read_csv(train_merger)
print('train_label_csv : ' +str(train_labels.shape[0]))
#adding the path for each id for easier processing
train_labels['path'] = train_labels['id'].apply(lambda x: f'../input/seti-breakthrough-listen/train/{x[0]}/{x}.npy')
train_labels.head()
"""
数据增强
"""
import albumentations
from albumentations.pytorch.transforms import ToTensorV2
from typing import *
class Transform:
def __init__(self, aug_kwargs: Dict):
albumentations_aug = [getattr(A, name)(**kwargs)
for name, kwargs in aug_kwargs.items()]
albumentations_aug.append(ToTensorV2(p=1))
self.transform = A.Compose(albumentations_aug)
def __call__(self, image):
image = self.transform(image = image)['image']
return image
class ModeTransform():
def __init__(self, df_frame, config, channel_mode,mode,target,transform):
self.df_frame = df_frame
self.channel_mode = channel_mode
self.config = config
self.target = target
self.file_names = df_frame['path'].values
self.labels = df_frame['target'].values
self.transform = transform
self.mode = mode
def __len__(self):
return len(self.df_frame)
def __getitem__(self, idx):
image = np.load(self.file_names[idx])
# print(image.shape) -> (6, 273, 256)
if self.channel_mode == 'spatial_6ch':
image = image.astype(np.float32)
image = np.vstack(image) # no transpose here (1638, 256)
#image = np.vstack(image).transpose((1, 0))
# print(image.shape) -> (256, 1638)
elif self.channel_mode == 'spatial_3ch':
image = image[::2].astype(np.float32)
image = np.vstack(image).transpose((1, 0))
elif self.channel_mode == '6_channel':
image = image.astype(np.float32)
image = np.transpose(image, (1,2,0))
elif self.channel_mode == '3_channel':
image = image[::2].astype(np.float32)
image = np.transpose(image, (1,2,0))
if self.transform:
image = self.transform(image)
else:
image = torch.from_numpy(image).float()
if self.mode == 'test':
return image
else:
label = torch.tensor(self.labels[idx]).float()
return image, label
import albumentations as A
CONFIG = {
"TRAIN_TRANSFORMS": {
"VerticalFlip": {"p": 0.5},
"HorizontalFlip": {"p": 0.5},
"Resize": {"height": 640, "width": 640, "p": 1},
}}
config = CONFIG
# Parameters
params_train = {'mode' : 'train',
'channel_mode' : 'spatial_6ch',
'target' : True}
train_dset = ModeTransform(train_labels,config,
**params_train,
transform=Transform(config["TRAIN_TRANSFORMS"]))
for i in range(2):
image, label = train_dset[i]
plt.imshow(image[0])
plt.title(f'label: {label}')
plt.show()
image.shape
"""
数据划分等参数设置
"""
class SETIDataset(tf.keras.utils.Sequence):
def __init__(self,df, directory, batch_size, random_state, shuffle, target):
np.random.seed(random_state)
self.directory = directory
self.df = df
self.target = target
self.shuffle = shuffle
self.batch_size = batch_size
self.ext = '.npy'
self.on_epoch_end()
def __len__(self):
len_ = np.ceil(self.df.shape[0] / self.batch_size).astype(int)
return len_
def __getitem__(self, idx):
start_idx = idx * self.batch_size
batch = self.df[start_idx: start_idx + self.batch_size]
signals = []
for fname in batch.id:
path = os.path.join(self.directory, fname[0], fname + self.ext)
data = np.load(path)
signals.append(data)
signals = np.transpose(np.stack(signals), (0, 1, 3, 2)).astype('float32')
if self.target:
return signals, batch.target.values
else:
return signals
def on_epoch_end(self):
if self.shuffle:
self.df = self.df.sample(frac=1).reset_index(drop=True)
train = pd.read_csv('../input/seti-breakthrough-listen/train_labels.csv')
sub = pd.read_csv('../input/seti-breakthrough-listen/sample_submission.csv')
sample_df = train.sample(frac=1).reset_index(drop=True)
split = int(sample_df.shape[0] * 0.8)
train_df = sample_df[:split]
valid_df = sample_df[split:]
# Parameters
params_train = {'batch_size' : 64,
'shuffle' : True,
'random_state' : 42,
'target' : True}
params_valid = {'batch_size' : 64,
'shuffle' : False,
'random_state' : 42,
'target' : True}
params_test = {'batch_size' : 64,
'shuffle' : False,
'random_state' : 42,
'target' : False}
train_dset = SETIDataset(
train_df, "../input/seti-breakthrough-listen/train", **params_train )
valid_dset = SETIDataset(
valid_df, "../input/seti-breakthrough-listen/train", **params_valid)
test_dset = SETIDataset(
sub, "../input/seti-breakthrough-listen/test", **params_test)
"""
定义模型
"""
def build_model(unit):
inputs = layers.Input(shape=(6, 256, 273))
gru1 = layers.Bidirectional(layers.GRU(unit, return_sequences = True))
gru2 = layers.Bidirectional(layers.GRU(unit, return_sequences = True))
pool = layers.GlobalAveragePooling1D()
model = layers.TimeDistributed(gru1, name="bi_gru_1")(inputs)
model = layers.TimeDistributed(gru2, name="bi_gru_2")(model)
model = layers.TimeDistributed(pool, name="pool")(model)
model = layers.Flatten()(model)
model = layers.Dense(128, activation="relu")(model)
model = layers.Dense(1, activation="sigmoid", name="sigmoid")(model)
model = models.Model(inputs = inputs, outputs = model)
model.compile("adam",
loss="binary_crossentropy",
metrics=[tf.keras.metrics.AUC()])
model.summary()
return model
"""
训练模型
"""
model = build_model(unit = 128)
model_save = ModelCheckpoint("model_weights.h5",
save_best_only=True,
save_weights_only=True)
history = model.fit(train_dset,
use_multiprocessing=True,
workers=4,
epochs=10,
validation_data=valid_dset,
callbacks=[model_save])
"""
训练过程loss可视化
"""
acc = history.history['auc']
val_acc = history.history['val_auc']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(acc) + 1)
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 5))
sns.set_style("white")
plt.suptitle('Train history', size = 15)
ax1.plot(epochs, acc, "bo", label = "Training acc")
ax1.plot(epochs, val_acc, "b", label = "Validation acc")
ax1.set_title("Training and validation acc")
ax1.legend()
ax2.plot(epochs, loss, "bo", label = "Training loss", color = 'red')
ax2.plot(epochs, val_loss, "b", label = "Validation loss", color = 'red')
ax2.set_title("Training and validation loss")
ax2.legend()
plt.show()
"""
模型推理与结果生成
"""
model.load_weights('model_weights.h5')
y_pred = model.predict(
test_dset,
use_multiprocessing=True,
workers=4,
verbose=1)
sub['target'] = y_pred
sub.to_csv('submission.csv', index=False)
sub.head()
"""
参考
https://keras.io/api/layers/recurrent_layers/gru/
"""