如何将Kaggle图片文件快速分类

题目

如何将Kaggle图片文件快速分类

前言

如果还不知道如何使用谷歌云盘下载Kaggle数据集+解压的小伙伴可以参考我的文章来教你解压！

下载完数据集解压后的文件分布，本文是拿dog-breed-identification来举例的

步骤

确定种类数

• 输出各个种类的狗的数量

import pandas as pd

dog = pd.read_csv('./labels.csv')['breed']
dog_breeds = []
for d in dog:
    if d not in dog_breeds:
      dog_breeds.append(d)
dog_breeds
dog_breeds_num = len(dog_breeds)
dog_breeds_num

# print(dog[1])
num = dict()
for i in range(len(dog)):
  num[dog[i]] = num.get(dog[i],0)+1
num

部分截图

创建种类文件夹

• 创建train文件夹所有狗种类文件夹

import os
path = '/content/gdrive/My Drive/demo'
for d_b in dog_breeds:
    os.mkdir(os.path.join(path,'train',d_b))

• 创建valid文件夹所有狗种类文件夹

os.mkdir(os.path.join(path,'valid'))
for d_b in dog_breeds:
    os.mkdir(os.path.join(path,'valid',d_b))

• 创建test文件夹未知文件夹

os.mkdir(os.path.join(path,'test','unknown'))

将文件移动到对应种类文件夹

• train

import shutil
dog_pic = pd.read_csv('./labels.csv')
for i in range(len(dog_pic)):
  shutil.move(os.path.join(path,'train',dog_pic.iloc[i,0]+'.jpg'),os.path.join(path,'train',dog_pic.iloc[i,1]))

• test

test_pic = os.listdir(os.path.join(path,'test'))
for t_p in test_pic:
  if t_p.endswith('.jpg'):
    shutil.move(os.path.join(path,'test',t_p),os.path.join(path,'test','unknown'))

• valid【将train中的一部分数据移动到valid中】

import random
train_dog_breeds = os.listdir(os.path.join(path,'train'))
# train_dog_breeds
for t_d_b in train_dog_breeds:
  one_dog_kind = os.listdir(os.path.join(path,'train',t_d_b))
  random.shuffle(one_dog_kind)
  vaild_data = random.sample(one_dog_kind, int(len(one_dog_kind)*0.2))
  for v_d in vaild_data:
    shutil.move(os.path.join(path,'train',t_d_b,v_d),os.path.join(path,'valid',t_d_b))

创建dataset

import torchvision
from torchvision import transforms
import torchvision
path = '/content/gdrive/My Drive/demo'

normalize = transforms.Normalize(
    mean=[0.485, 0.456, 0.406],
    std=[0.229, 0.224, 0.225]
)
transform_train = transforms.Compose([
    transforms.RandomResizedCrop(
        size=224,
        scale=(0.08, 1),
        ratio=(3.0/4.0, 4.0/3.0),
    ),
    transforms.RandomHorizontalFlip(),
    transforms.ColorJitter(
        brightness=0.4,
        contrast=0.4,
        saturation=0.4
    ),
    transforms.ToTensor(),
    normalize
])

transform_test = transforms.Compose([
    transforms.Resize(256),
    transforms.CenterCrop(224),
    transforms.ToTensor(),
    normalize
])

train_ds, valid_ds = [
    torchvision.datasets.ImageFolder(
        root='/'.join([path,folder]),
        transform=transform_train
    ) for folder in ['train', 'valid']
]

test_ds = torchvision.datasets.ImageFolder(
    root='/'.join([path,'test']),
    transform=transform_test
)
# train_ds.class_to_idx,train_ds.imgs

创建dataloader

from torch.utils.data import DataLoader
BATCH_SIZE = 32
train_dataloader = DataLoader(
    dataset=train_ds,
    batch_size=BATCH_SIZE,
    shuffle=True,
    num_workers=2,
    drop_last=True
)


valid_dataloader = DataLoader(
    dataset=valid_ds,
    batch_size=BATCH_SIZE,
    shuffle=True,
    num_workers=2,
    drop_last=True
)

   
test_dataloader = DataLoader(
    dataset=test_ds,
    batch_size=BATCH_SIZE,
    shuffle=False,
    num_workers=2,
    drop_last=True
)
len(train_dataloader),len(valid_dataloader),len(test_dataloader)

查看是否创建dataloader成功

import numpy as np
import matplotlib.pyplot as plt
image, label = iter(test_dataloader).next()
sample = image[0].squeeze()
sample = sample.permute((1, 2, 0)).numpy()
sample *= [0.229, 0.224, 0.225]
sample += [0.485, 0.456, 0.406]
sample = np.clip(sample, 0, 1)
plt.imshow(sample)
plt.show()
print('Label is: {}'.format(label[0].numpy()))

后记

本来是想练习cnn模型的，结果训练结果太差，不到10%，哎，太难了。。。上网一查，得用更为复杂的模型，算了，我又不是CV专业😂