MMPreTrain实战记录

---

课程总结

mim工具可以在任意位置启动训练，永远滴神！

自定义配置文件的注意事项

1. 修改数据集类别+训练、验证、测试集路径
2. 调整学习率

实战记录

配置环境

!git clone https://github.com/open-mmlab/mmpretrain.git
%cd mmpretrain
!pip install -U openmim && mim install -e .

获取完整配置文件

原理：即使在训练时因为数据集不存在而无法成功运行也会生成完整的配置文件

!python /content/mmpretrain/tools/train.py /content/mmpretrain/configs/convmixer/convmixer-768-32_10xb64_in1k.py

数据集下载+解压

!gdown https://drive.google.com/uc?id=16AvoLcKzCfvsKFRVQ-RGVnsTRWzAe7ze 

import tarfile

tar_path = "/content/mmpretrain/fruit30_train.tar"  # Path to your tar file
destination_path = "/content/mmpretrain/data/fruits"  # Path to the destination folder

with tarfile.open(tar_path) as tar:
    tar.extractall(path=destination_path)

数据集类别数确认

import os

folder_path = "/content/mmpretrain/data/fruits"  # replace with your folder path
items = os.listdir(folder_path)

folder_count = sum([1 for item in items if os.path.isdir(os.path.join(folder_path, item))])

print("Number of folders:", folder_count)

划分数据集

import os
import shutil
import random

# 定义路径
data_path = "/content/mmpretrain/data/fruits"  # 源数据路径
train_path = "./data/train"  # 训练集路径
valid_path = "./data/valid"  # 验证集路径
test_path = "./data/test"  # 测试集路径

# 创建train, valid, test文件夹
os.makedirs(train_path, exist_ok=True)
os.makedirs(valid_path, exist_ok=True)
os.makedirs(test_path, exist_ok=True)

# 遍历data文件夹下的每个子文件夹
for folder in os.listdir(data_path):
    folder_path = os.path.join(data_path, folder)
    if os.path.isdir(folder_path):
        # 创建相应的子文件夹
        os.makedirs(os.path.join(train_path, folder), exist_ok=True)
        os.makedirs(os.path.join(valid_path, folder), exist_ok=True)
        os.makedirs(os.path.join(test_path, folder), exist_ok=True)

        # 获取子文件夹中的文件列表
        files = os.listdir(folder_path)
        # 打乱文件顺序
        random.shuffle(files)

        # 计算训练、验证、测试集的文件数量
        train_count = int(len(files) * 0.7)
        valid_count = int(len(files) * 0.2)
        test_count = len(files) - train_count - valid_count

        # 分别划分训练、验证、测试集
        train_files = files[:train_count]
        valid_files = files[train_count:(train_count+valid_count)]
        test_files = files[(train_count+valid_count):]

        # 移动文件
        for file in train_files:
            shutil.move(os.path.join(folder_path, file), os.path.join(train_path, folder, file))
        for file in valid_files:
            shutil.move(os.path.join(folder_path, file), os.path.join(valid_path, folder, file))
        for file in test_files:
            shutil.move(os.path.join(folder_path, file), os.path.join(test_path, folder, file))

修改配置文件

cfg="""
model = dict(
    type='ImageClassifier',
    backbone=dict(type='ConvMixer', arch='768/32', act_cfg=dict(type='ReLU')),
    neck=dict(type='GlobalAveragePooling'),
    head=dict(
        type='LinearClsHead',
        num_classes=30,
        in_channels=768,
        loss=dict(type='CrossEntropyLoss', loss_weight=1.0)))
dataset_type = 'CustomDataset'
data_preprocessor = dict(
    num_classes=30,
    mean=[123.675, 116.28, 103.53],
    std=[58.395, 57.12, 57.375],
    to_rgb=True)
bgr_mean = [103.53, 116.28, 123.675]
bgr_std = [57.375, 57.12, 58.395]
train_pipeline = [
    dict(type='LoadImageFromFile'),
    dict(
        type='RandomResizedCrop',
        scale=224,
        backend='pillow',
        interpolation='bicubic'),
    dict(type='RandomFlip', prob=0.5, direction='horizontal'),
    dict(
        type='RandAugment',
        policies='timm_increasing',
        num_policies=2,
        total_level=10,
        magnitude_level=9,
        magnitude_std=0.5,
        hparams=dict(pad_val=[104, 116, 124], interpolation='bicubic')),
    dict(
        type='RandomErasing',
        erase_prob=0.25,
        mode='rand',
        min_area_ratio=0.02,
        max_area_ratio=0.3333333333333333,
        fill_color=[103.53, 116.28, 123.675],
        fill_std=[57.375, 57.12, 58.395]),
    dict(type='PackInputs')
]
test_pipeline = [
    dict(type='LoadImageFromFile'),
    dict(
        type='ResizeEdge',
        scale=233,
        edge='short',
        backend='pillow',
        interpolation='bicubic'),
    dict(type='CenterCrop', crop_size=224),
    dict(type='PackInputs')
]
train_dataloader = dict(
    pin_memory=True,
    persistent_workers=True,
    collate_fn=dict(type='default_collate'),
    batch_size=32,
    num_workers=5,
    dataset=dict(
        type='CustomDataset',
        data_root='/content/mmpretrain/data/train',
        pipeline=[
            dict(type='LoadImageFromFile'),
            dict(
                type='RandomResizedCrop',
                scale=224,
                backend='pillow',
                interpolation='bicubic'),
            dict(type='RandomFlip', prob=0.5, direction='horizontal'),
            dict(
                type='RandAugment',
                policies='timm_increasing',
                num_policies=2,
                total_level=10,
                magnitude_level=9,
                magnitude_std=0.5,
                hparams=dict(pad_val=[104, 116, 124],
                             interpolation='bicubic')),
            dict(
                type='RandomErasing',
                erase_prob=0.25,
                mode='rand',
                min_area_ratio=0.02,
                max_area_ratio=0.3333333333333333,
                fill_color=[103.53, 116.28, 123.675],
                fill_std=[57.375, 57.12, 58.395]),
            dict(type='PackInputs')
        ]),
    sampler=dict(type='DefaultSampler', shuffle=True))
val_dataloader = dict(
    pin_memory=True,
    persistent_workers=True,
    collate_fn=dict(type='default_collate'),
    batch_size=32,
    num_workers=5,
    dataset=dict(
        type='CustomDataset',
        data_root='/content/mmpretrain/data/valid',
        pipeline=[
            dict(type='LoadImageFromFile'),
            dict(
                type='ResizeEdge',
                scale=233,
                edge='short',
                backend='pillow',
                interpolation='bicubic'),
            dict(type='CenterCrop', crop_size=224),
            dict(type='PackInputs')
        ]),
    sampler=dict(type='DefaultSampler', shuffle=False))
val_evaluator = dict(type='Accuracy', topk=1)
test_dataloader = dict(
    pin_memory=True,
    persistent_workers=True,
    collate_fn=dict(type='default_collate'),
    batch_size=32,
    num_workers=5,
    dataset=dict(
        type='CustomDataset',
        data_root='/content/mmpretrain/data/test',
        pipeline=[
            dict(type='LoadImageFromFile'),
            dict(
                type='ResizeEdge',
                scale=233,
                edge='short',
                backend='pillow',
                interpolation='bicubic'),
            dict(type='CenterCrop', crop_size=224),
            dict(type='PackInputs')
        ]),
    sampler=dict(type='DefaultSampler', shuffle=False))
test_evaluator = dict(type='Accuracy', topk=1)
optim_wrapper = dict(
    optimizer=dict(
        type='AdamW',
        lr=0.01,
        weight_decay=0.05,
        eps=1e-08,
        betas=(0.9, 0.999)),
    paramwise_cfg=dict(
        norm_decay_mult=0.0,
        bias_decay_mult=0.0,
        flat_decay_mult=0.0,
        custom_keys=dict({
            '.absolute_pos_embed': dict(decay_mult=0.0),
            '.relative_position_bias_table': dict(decay_mult=0.0)
        })),
    clip_grad=dict(max_norm=5.0))
param_scheduler = [
    dict(
        type='LinearLR',
        start_factor=0.001,
        by_epoch=True,
        end=20,
        convert_to_iter_based=True),
    dict(type='CosineAnnealingLR', eta_min=1e-05, by_epoch=True, begin=20)
]
train_cfg = dict(by_epoch=True, max_epochs=300, val_interval=1)
val_cfg = dict()
test_cfg = dict()
auto_scale_lr = dict(base_batch_size=640)
default_scope = 'mmpretrain'
default_hooks = dict(
    timer=dict(type='IterTimerHook'),
    logger=dict(type='LoggerHook', interval=5),
    param_scheduler=dict(type='ParamSchedulerHook'),
    checkpoint=dict(type='CheckpointHook', interval=1),
    sampler_seed=dict(type='DistSamplerSeedHook'),
    visualization=dict(type='VisualizationHook', enable=False))
env_cfg = dict(
    cudnn_benchmark=False,
    mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0),
    dist_cfg=dict(backend='nccl'))
vis_backends = [dict(type='LocalVisBackend')]
visualizer = dict(
    type='UniversalVisualizer', vis_backends=[dict(type='LocalVisBackend')])
log_level = 'INFO'
load_from = 'https://download.openmmlab.com/mmclassification/v0/convmixer/convmixer-768-32_3rdparty_10xb64_in1k_20220323-bca1f7b8.pth'
resume = False
randomness = dict(seed=None, deterministic=False)
launcher = 'none'
work_dir = './work_dirs/convmixer-768-32_10xb64_in1k'
"""
with open('/content/mmpretrain/work_dirs/convmixer-768-32_10xb64_in1k/convmixer-768-32_10xb64_in1k.py', 'w') as f: 
    f.write(cfg)

训练

!python /content/mmpretrain/tools/train.py /content/mmpretrain/work_dirs/convmixer-768-32_10xb64_in1k/convmixer-768-32_10xb64_in1k.py

测试

!python /content/mmpretrain/tools/test.py /content/mmpretrain/work_dirs/convmixer-768-32_10xb64_in1k/convmixer-768-32_10xb64_in1k.py /content/mmpretrain/work_dirs/convmixer-768-32_10xb64_in1k/epoch_3.pth

推理

!python /content/mmpretrain/demo/image_demo.py /content/mmpretrain/data/test/哈密瓜/116.jpg \
/content/mmpretrain/work_dirs/convmixer-768-32_10xb64_in1k/convmixer-768-32_10xb64_in1k.py \
--checkpoint /content/mmpretrain/work_dirs/convmixer-768-32_10xb64_in1k/epoch_3.pth --show --show-dir ./