Kaggle比赛地址
https://www.kaggle.com/competitions/classify-leaves/leaderboard
代码
# 首先导入包
import torch
import torch.nn as nn
import pandas as pd
import numpy as np
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms
from PIL import Image
import os
import matplotlib.pyplot as plt
import torchvision.models as models
# This is for the progress bar.
from tqdm import tqdm
# 看看label文件长啥样
labels_dataframe = pd.read_csv('./input0/train.csv')
#labels_dataframe.head(5)
# 把label文件排个序
leaves_labels = sorted(list(set(labels_dataframe['label'])))
n_classes = len(leaves_labels)
#print(n_classes)
#leaves_labels[:10]
# 把label转成对应的数字
class_to_num = dict(zip(leaves_labels, range(n_classes)))
#class_to_num
# 再转换回来，方便最后预测的时候使用
num_to_class = {v : k for k, v in class_to_num.items()}
# 继承pytorch的dataset，创建自己的
class LeavesData(Dataset):
def __init__(self, csv_path, file_path, mode='train', valid_ratio=0.2, resize_height=256, resize_width=256):
"""
Args:
csv_path (string): csv 文件路径
img_path (string): 图像文件所在路径
mode (string): 训练模式还是测试模式
valid_ratio (float): 验证集比例
"""
# 需要调整后的照片尺寸，我这里每张图片的大小尺寸不一致#
self.resize_height = resize_height
self.resize_width = resize_width
self.file_path = file_path
self.mode = mode
# 读取 csv 文件
# 利用pandas读取csv文件
self.data_info = pd.read_csv(csv_path, header=None) #header=None是去掉表头部分
# 计算 length
self.data_len = len(self.data_info.index) - 1
self.train_len = int(self.data_len * (1 - valid_ratio))
if mode == 'train':
# 第一列包含图像文件的名称
self.train_image = np.asarray(self.data_info.iloc[1:self.train_len, 0]) #self.data_info.iloc[1:,0]表示读取第一列，从第二行开始到train_len
# 第二列是图像的 label
self.train_label = np.asarray(self.data_info.iloc[1:self.train_len, 1])
self.image_arr = self.train_image
self.label_arr = self.train_label
elif mode == 'valid':
self.valid_image = np.asarray(self.data_info.iloc[self.train_len:, 0])
self.valid_label = np.asarray(self.data_info.iloc[self.train_len:, 1])
self.image_arr = self.valid_image
self.label_arr = self.valid_label
elif mode == 'test':
self.test_image = np.asarray(self.data_info.iloc[1:, 0])
self.image_arr = self.test_image
self.real_len = len(self.image_arr)
print('Finished reading the {} set of Leaves Dataset ({} samples found)'
.format(mode, self.real_len))
def __getitem__(self, index):
# 从 image_arr中得到索引对应的文件名
single_image_name = self.image_arr[index]
# 读取图像文件
img_as_img = Image.open(self.file_path + single_image_name)
#如果需要将RGB三通道的图片转换成灰度图片可参考下面两行
# if img_as_img.mode != 'L':
# img_as_img = img_as_img.convert('L')
#设置好需要转换的变量，还可以包括一系列的nomarlize等等操作
if self.mode == 'train':
transform = transforms.Compose([
transforms.Resize((224, 224)),
transforms.RandomHorizontalFlip(p=0.5), #随机水平翻转 选择一个概率
#transforms.RandomResizedCrop(size=224, scale=(0.8, 1.0)),
#transforms.RandomRotation(degrees=15),
#transforms.RandomHorizontalFlip(),
#transforms.CenterCrop(size=224),
transforms.ToTensor(),
#transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])
])
else:
# valid和test不做数据增强
transform = transforms.Compose([
transforms.Resize((224, 224)),
transforms.CenterCrop(size=224),
transforms.ToTensor(),
#transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])
])
img_as_img = transform(img_as_img)
if self.mode == 'test':
return img_as_img
else:
# 得到图像的 string label
label = self.label_arr[index]
# number label
number_label = class_to_num[label]
return img_as_img, number_label #返回每一个index对应的图片数据和对应的label
def __len__(self):
return self.real_len
train_path = './input0/train.csv'
test_path = './input0/test.csv'
# csv文件中已经images的路径了，因此这里只到上一级目录
img_path = './input0/'
train_dataset = LeavesData(train_path, img_path, mode='train')
val_dataset = LeavesData(train_path, img_path, mode='valid')
test_dataset = LeavesData(test_path, img_path, mode='test')
#print(train_dataset)
#print(val_dataset)
#print(test_dataset)
Finished reading the train set of Leaves Dataset (14681 samples found)
Finished reading the valid set of Leaves Dataset (3672 samples found)
Finished reading the test set of Leaves Dataset (8800 samples found)
# 定义data loader
train_loader = torch.utils.data.DataLoader(
dataset=train_dataset,
batch_size=32,
shuffle=False,
num_workers=5
)
val_loader = torch.utils.data.DataLoader(
dataset=val_dataset,
batch_size=16,
shuffle=False,
num_workers=5
)
test_loader = torch.utils.data.DataLoader(
dataset=test_dataset,
batch_size=16,
shuffle=False,
num_workers=5
)
# 给大家展示一下数据长啥样
def im_convert(tensor):
""" 展示数据"""
image = tensor.to("cpu").clone().detach()
image = image.numpy().squeeze()
image = image.transpose(1,2,0)
image = image.clip(0, 1)
return image
fig=plt.figure(figsize=(20, 12))
columns = 4
rows = 2
dataiter = iter(val_loader)
inputs, classes = dataiter.next()
for idx in range (columns*rows):
ax = fig.add_subplot(rows, columns, idx+1, xticks=[], yticks=[])
ax.set_title(num_to_class[int(classes[idx])])
plt.imshow(im_convert(inputs[idx]))
plt.show()
# 看一下是在cpu还是GPU上
def get_device():
return 'cuda' if torch.cuda.is_available() else 'cpu'
device = get_device()
print(device)
cuda
#from efficientnet_pytorch import EfficientNet
# 是否要冻住模型的前面一些层
def set_parameter_requires_grad(model, feature_extracting):
if feature_extracting:
model = model
for param in model.parameters():
param.requires_grad = False
# resnet34模型
def res_model(num_classes, feature_extract = False, use_pretrained=True):
model_ft = models.resnet50(pretrained=use_pretrained)
set_parameter_requires_grad(model_ft, feature_extract)
num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Sequential(nn.Linear(num_ftrs, num_classes))
#model = EfficientNet.from_name('efficientnet-b3')
#model.load_state_dict(torch.load('./adv-efficientnet-b3-cdd7c0f4.pth'))
#fc_features = model._fc.in_features
#model._fc = nn.Linear(fc_features, num_classes)
return model_ft
# 超参数, 这里为了演示就训练5轮看看
learning_rate = 2e-5
weight_decay = 1e-3
num_epoch = 18
model_path = './pre_res_model.ckpt'
# Initialize a model, and put it on the device specified.
model = res_model(176)
model = model.to(device)
model.device = device
# For the classification task, we use cross-entropy as the measurement of performance.
criterion = nn.CrossEntropyLoss()
# Initialize optimizer, you may fine-tune some hyperparameters such as learning rate on your own.
optimizer = torch.optim.Adam(model.parameters(), lr = learning_rate, weight_decay=weight_decay)
# The number of training epochs.
n_epochs = num_epoch
best_acc = 0.0
for epoch in range(n_epochs):
# ---------- Training ----------
# Make sure the model is in train mode before training.
model.train()
# These are used to record information in training.
train_loss = []
train_accs = []
# Iterate the training set by batches.
for batch in tqdm(train_loader):
# A batch consists of image data and corresponding labels.
imgs, labels = batch
imgs = imgs.to(device)
labels = labels.to(device)
# Forward the data. (Make sure data and model are on the same device.)
logits = model(imgs)
# Calculate the cross-entropy loss.
# We don't need to apply softmax before computing cross-entropy as it is done automatically.
loss = criterion(logits, labels)
# Gradients stored in the parameters in the previous step should be cleared out first.
optimizer.zero_grad()
# Compute the gradients for parameters.
loss.backward()
# Update the parameters with computed gradients.
optimizer.step()
# Compute the accuracy for current batch.
acc = (logits.argmax(dim=-1) == labels).float().mean()
# Record the loss and accuracy.
train_loss.append(loss.item())
train_accs.append(acc)
# The average loss and accuracy of the training set is the average of the recorded values.
train_loss = sum(train_loss) / len(train_loss)
train_acc = sum(train_accs) / len(train_accs)
# Print the information.
print(f"[ Train | {epoch + 1:03d}/{n_epochs:03d} ] loss = {train_loss:.5f}, acc = {train_acc:.5f}")
# ---------- Validation ----------
# Make sure the model is in eval mode so that some modules like dropout are disabled and work normally.
model.eval()
# These are used to record information in validation.
valid_loss = []
valid_accs = []
# Iterate the validation set by batches.
for batch in tqdm(val_loader):
imgs, labels = batch
# We don't need gradient in validation.
# Using torch.no_grad() accelerates the forward process.
with torch.no_grad():
logits = model(imgs.to(device))
# We can still compute the loss (but not the gradient).
loss = criterion(logits, labels.to(device))
# Compute the accuracy for current batch.
acc = (logits.argmax(dim=-1) == labels.to(device)).float().mean()
# Record the loss and accuracy.
valid_loss.append(loss.item())
valid_accs.append(acc)
# The average loss and accuracy for entire validation set is the average of the recorded values.
valid_loss = sum(valid_loss) / len(valid_loss)
valid_acc = sum(valid_accs) / len(valid_accs)
# Print the information.
print(f"[ Valid | {epoch + 1:03d}/{n_epochs:03d} ] loss = {valid_loss:.5f}, acc = {valid_acc:.5f}")
# if the model improves, save a checkpoint at this epoch
if valid_acc > best_acc:
best_acc = valid_acc
torch.save(model.state_dict(), model_path)
print('saving model with acc {:.3f}'.format(best_acc))
100%|██████████| 459/459 [01:27<00:00, 5.24it/s]
0%| | 0/230 [00:00<?, ?it/s]
[ Train | 001/018 ] loss = 4.02834, acc = 0.21657
100%|██████████| 230/230 [00:13<00:00, 17.60it/s]
[ Valid | 001/018 ] loss = 2.79563, acc = 0.39049
0%| | 0/459 [00:00<?, ?it/s]
saving model with acc 0.390
100%|██████████| 459/459 [01:27<00:00, 5.26it/s]
0%| | 0/230 [00:00<?, ?it/s]
[ Train | 002/018 ] loss = 2.12002, acc = 0.56044
100%|██████████| 230/230 [00:08<00:00, 26.57it/s]
[ Valid | 002/018 ] loss = 1.50455, acc = 0.66848
0%| | 0/459 [00:00<?, ?it/s]
saving model with acc 0.668
100%|██████████| 459/459 [01:27<00:00, 5.25it/s]
0%| | 0/230 [00:00<?, ?it/s]
[ Train | 003/018 ] loss = 1.18445, acc = 0.77650
100%|██████████| 230/230 [00:08<00:00, 26.51it/s]
[ Valid | 003/018 ] loss = 0.89174, acc = 0.80951
0%| | 0/459 [00:00<?, ?it/s]
saving model with acc 0.810
100%|██████████| 459/459 [01:27<00:00, 5.23it/s]
0%| | 0/230 [00:00<?, ?it/s]
[ Train | 004/018 ] loss = 0.67371, acc = 0.88586
100%|██████████| 230/230 [00:08<00:00, 26.57it/s]
[ Valid | 004/018 ] loss = 0.57501, acc = 0.87147
0%| | 0/459 [00:00<?, ?it/s]
saving model with acc 0.871
100%|██████████| 459/459 [01:27<00:00, 5.25it/s]
0%| | 0/230 [00:00<?, ?it/s]
[ Train | 005/018 ] loss = 0.38276, acc = 0.94710
100%|██████████| 230/230 [00:08<00:00, 26.59it/s]
[ Valid | 005/018 ] loss = 0.42275, acc = 0.89728
0%| | 0/459 [00:00<?, ?it/s]
saving model with acc 0.897
100%|██████████| 459/459 [01:28<00:00, 5.21it/s]
0%| | 0/230 [00:00<?, ?it/s]
[ Train | 006/018 ] loss = 0.22507, acc = 0.97350
100%|██████████| 230/230 [00:08<00:00, 26.64it/s]
[ Valid | 006/018 ] loss = 0.36002, acc = 0.90761
0%| | 0/459 [00:00<?, ?it/s]
saving model with acc 0.908
100%|██████████| 459/459 [01:27<00:00, 5.22it/s]
0%| | 0/230 [00:00<?, ?it/s]
[ Train | 007/018 ] loss = 0.13652, acc = 0.98638
100%|██████████| 230/230 [00:08<00:00, 26.50it/s]
[ Valid | 007/018 ] loss = 0.32017, acc = 0.91413
0%| | 0/459 [00:00<?, ?it/s]
saving model with acc 0.914
100%|██████████| 459/459 [01:27<00:00, 5.23it/s]
0%| | 0/230 [00:00<?, ?it/s]
[ Train | 008/018 ] loss = 0.09041, acc = 0.99059
100%|██████████| 230/230 [00:08<00:00, 26.40it/s]
[ Valid | 008/018 ] loss = 0.29222, acc = 0.92120
0%| | 0/459 [00:00<?, ?it/s]
saving model with acc 0.921
100%|██████████| 459/459 [01:27<00:00, 5.23it/s]
0%| | 0/230 [00:00<?, ?it/s]
[ Train | 009/018 ] loss = 0.06629, acc = 0.99203
100%|██████████| 230/230 [00:11<00:00, 19.67it/s]
0%| | 0/459 [00:00<?, ?it/s]
[ Valid | 009/018 ] loss = 0.31396, acc = 0.91766
100%|██████████| 459/459 [01:28<00:00, 5.19it/s]
0%| | 0/230 [00:00<?, ?it/s]
[ Train | 010/018 ] loss = 0.05238, acc = 0.99374
100%|██████████| 230/230 [00:08<00:00, 26.24it/s]
[ Valid | 010/018 ] loss = 0.29013, acc = 0.92174
0%| | 0/459 [00:00<?, ?it/s]
saving model with acc 0.922
100%|██████████| 459/459 [01:27<00:00, 5.24it/s]
0%| | 0/230 [00:00<?, ?it/s]
[ Train | 011/018 ] loss = 0.04813, acc = 0.99285
100%|██████████| 230/230 [00:08<00:00, 26.72it/s]
[ Valid | 011/018 ] loss = 0.28971, acc = 0.92283
0%| | 0/459 [00:00<?, ?it/s]
saving model with acc 0.923
100%|██████████| 459/459 [01:27<00:00, 5.24it/s]
0%| | 0/230 [00:00<?, ?it/s]
[ Train | 012/018 ] loss = 0.04135, acc = 0.99367
100%|██████████| 230/230 [00:08<00:00, 26.08it/s]
0%| | 0/459 [00:00<?, ?it/s]
[ Valid | 012/018 ] loss = 0.31470, acc = 0.91739
100%|██████████| 459/459 [01:27<00:00, 5.23it/s]
0%| | 0/230 [00:00<?, ?it/s]
[ Train | 013/018 ] loss = 0.04344, acc = 0.99237
100%|██████████| 230/230 [00:08<00:00, 26.78it/s]
[ Valid | 013/018 ] loss = 0.26491, acc = 0.92826
0%| | 0/459 [00:00<?, ?it/s]
saving model with acc 0.928
100%|██████████| 459/459 [01:27<00:00, 5.22it/s]
0%| | 0/230 [00:00<?, ?it/s]
[ Train | 014/018 ] loss = 0.03887, acc = 0.99340
100%|██████████| 230/230 [00:08<00:00, 26.33it/s]
0%| | 0/459 [00:00<?, ?it/s]
[ Valid | 014/018 ] loss = 0.27231, acc = 0.92772
100%|██████████| 459/459 [01:27<00:00, 5.24it/s]
0%| | 0/230 [00:00<?, ?it/s]
[ Train | 015/018 ] loss = 0.02931, acc = 0.99455
100%|██████████| 230/230 [00:08<00:00, 26.36it/s]
0%| | 0/459 [00:00<?, ?it/s]
[ Valid | 015/018 ] loss = 0.30156, acc = 0.91875
100%|██████████| 459/459 [01:27<00:00, 5.25it/s]
0%| | 0/230 [00:00<?, ?it/s]
[ Train | 016/018 ] loss = 0.04074, acc = 0.99258
100%|██████████| 230/230 [00:08<00:00, 26.61it/s]
0%| | 0/459 [00:00<?, ?it/s]
[ Valid | 016/018 ] loss = 0.29693, acc = 0.92147
100%|██████████| 459/459 [01:27<00:00, 5.23it/s]
0%| | 0/230 [00:00<?, ?it/s]
[ Train | 017/018 ] loss = 0.03940, acc = 0.99428
100%|██████████| 230/230 [00:08<00:00, 26.39it/s]
[ Valid | 017/018 ] loss = 0.26381, acc = 0.93288
0%| | 0/459 [00:00<?, ?it/s]
saving model with acc 0.933
100%|██████████| 459/459 [01:27<00:00, 5.23it/s]
0%| | 0/230 [00:00<?, ?it/s]
[ Train | 018/018 ] loss = 0.02420, acc = 0.99619
100%|██████████| 230/230 [00:08<00:00, 26.32it/s]
[ Valid | 018/018 ] loss = 0.25977, acc = 0.93152
saveFileName = './submission.csv'
## predict
model = res_model(176)
# create model and load weights from checkpoint
model = model.to(device)
model.load_state_dict(torch.load(model_path))
# Make sure the model is in eval mode.
# Some modules like Dropout or BatchNorm affect if the model is in training mode.
model.eval()
# Initialize a list to store the predictions.
predictions = []
# Iterate the testing set by batches.
for batch in tqdm(test_loader):
imgs = batch
with torch.no_grad():
logits = model(imgs.to(device))
# Take the class with greatest logit as prediction and record it.
predictions.extend(logits.argmax(dim=-1).cpu().numpy().tolist())
preds = []
for i in predictions:
preds.append(num_to_class[i])
test_data = pd.read_csv(test_path)
test_data['label'] = pd.Series(preds)
submission = pd.concat([test_data['image'], test_data['label']], axis=1)
submission.to_csv(saveFileName, index=False)
100%|██████████| 550/550 [00:19<00:00, 28.43it/s]
2295
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
