[训练和优化] 2. 超参数调优-CSDN博客

本文链接：https://blog.csdn.net/ayiya_Oese/article/details/147939200

👋 你好！这里有实用干货与深度分享✨✨ 若有帮助，欢迎：
👍 点赞 | ⭐ 收藏 | 💬 评论 | ➕ 关注，解锁更多精彩！
📁 收藏专栏即可第一时间获取最新推送🔔。
📖后续我将持续带来更多优质内容，期待与你一同探索知识，携手前行，共同进步🚀。

人工智能

超参数调优

本文详细介绍深度学习模型的超参数调优方法，包括学习率调整、批量大小选择、优化器参数、超参数搜索等内容，帮助你高效、系统地优化模型性能。

1. 学习率调整

1.1 学习率调度器

灵活调整学习率以适应训练进程，常用调度器如下：

from torch.optim.lr_scheduler import ReduceLROnPlateau, CosineAnnealingLR

def create_lr_scheduler(optimizer, scheduler_type='plateau', **kwargs):
    if scheduler_type == 'plateau':
        return ReduceLROnPlateau(
            optimizer,
            mode='min',
            factor=0.1,
            patience=10,
            verbose=True
        )
    elif scheduler_type == 'cosine':
        return CosineAnnealingLR(
            optimizer,
            T_max=100,
            eta_min=1e-6
        )
    else:
        raise ValueError(f'Unsupported scheduler type: {scheduler_type}')


scheduler = create_lr_scheduler(optimizer, 'plateau')

1.2 学习率预热

训练初期采用较小学习率，逐步升高，提升收敛稳定性。

class WarmupScheduler:
    def __init__(self, optimizer, warmup_epochs, initial_lr, target_lr):
        self.optimizer = optimizer
        self.warmup_epochs = warmup_epochs
        self.initial_lr = initial_lr
        self.target_lr = target_lr
        self.current_epoch = 0
    
    def step(self):
        if self.current_epoch < self.warmup_epochs:
            lr = self.initial_lr + (self.target_lr - self.initial_lr) * \
                 (self.current_epoch / self.warmup_epochs)
            for param_group in self.optimizer.param_groups:
                param_group['lr'] = lr

        self.current_epoch += 1

2. 批量大小选择

2.1 动态批量大小

根据显存和训练进度动态调整 batch size。

class DynamicBatchSizer:
    def __init__(self, initial_batch_size, max_batch_size, growth_factor=1.5):
        self.current_batch_size = initial_batch_size
        self.max_batch_size = max_batch_size
        self.growth_factor = growth_factor
    
    def increase_batch_size(self):
        new_size = int(self.current_batch_size * self.growth_factor)
        self.current_batch_size = min(new_size, self.max_batch_size)
        return self.current_batch_size


batch_sizer = DynamicBatchSizer(32, 256)

2.2 梯度累积

通过多步累积梯度，模拟大批量训练，节省显存。

def train_with_gradient_accumulation(model, train_loader, criterion, optimizer,
                                    device, accumulation_steps):
    model.train()
    optimizer.zero_grad()

    for i, (data, target) in enumerate(train_loader):
        data, target = data.to(device), target.to(device)

        # 前向传播
        output = model(data)
        loss = criterion(output, target)

        # 缩放损失
        loss = loss / accumulation_steps

        # 反向传播
        loss.backward()

        # 累计指定步数后更新参数
        if (i + 1) % accumulation_steps == 0:
            optimizer.step()
            optimizer.zero_grad()

3. 优化器参数调优

3.1 优化器选择

根据任务特点选择合适的优化器及其参数。

import torch.optim as optim

def create_optimizer(model, optimizer_type='adam', **kwargs):
    if optimizer_type == 'adam':
        return optim.Adam(
            model.parameters(),
            lr=kwargs.get('lr', 0.001),
            weight_decay=kwargs.get('weight_decay', 0.0)
        )
    elif optimizer_type == 'sgd':
        return optim.SGD(
            model.parameters(),
            lr=kwargs.get('lr', 0.01),
            momentum=kwargs.get('momentum', 0.9),
            weight_decay=kwargs.get('weight_decay', 0.0)
        )
    else:
        raise ValueError(f'Unsupported optimizer type: {optimizer_type}')

3.2 权重衰减调整

为不同层设置不同的权重衰减，提升泛化能力。

def apply_weight_decay_by_layer(model, weight_decay_map):
    """为不同层设置不同的权重衰减"""
    parameter_groups = []

    for name, param in model.named_parameters():
        if not param.requires_grad:
            continue

        # 根据层名称设置权重衰减
        decay = 0.0
        for layer_name, decay_value in weight_decay_map.items():
            if layer_name in name:
                decay = decay_value
                break

        parameter_groups.append({
            'params': param,
            'weight_decay': decay
        })

    return parameter_groups


weight_decay_map = {
    'conv': 0.001,
    'linear': 0.01
}
params = apply_weight_decay_by_layer(model, weight_decay_map)
optimizer = optim.Adam(params)

4. 超参数搜索

4.1 网格搜索

遍历所有参数组合，找到最优超参数。

from itertools import product

def grid_search(param_grid, train_fn):
    """网格搜索超参数"""
    results = []

    # 生成所有参数组合
    param_combinations = [dict(zip(param_grid.keys(), v)) 
                        for v in product(*param_grid.values())]

    for params in param_combinations:
        # 训练模型并记录结果
        score = train_fn(**params)
        results.append({
            'params': params,
            'score': score
        })

    # 根据得分排序
    results.sort(key=lambda x: x['score'], reverse=True)
    return results


param_grid = {
    'lr': [0.1, 0.01, 0.001],
    'batch_size': [32, 64, 128],
    'weight_decay': [0.0, 0.001, 0.01]
}

4.2 随机搜索

随机采样参数组合，提升搜索效率。

import random

def random_search(param_ranges, n_iter, train_fn):
    """随机搜索超参数"""
    results = []

    for _ in range(n_iter):

        params = {}
        for param_name, param_range in param_ranges.items():
            if isinstance(param_range[0], float):
                params[param_name] = random.uniform(*param_range)
            else:
                params[param_name] = random.randint(*param_range)

        # 训练模型并记录结果
        score = train_fn(**params)
        results.append({
            'params': params,
            'score': score
        })

    # 根据得分排序
    results.sort(key=lambda x: x['score'], reverse=True)
    return results


param_ranges = {
    'lr': [1e-4, 1e-2],
    'batch_size': [16, 256],
    'weight_decay': [0.0, 0.1]
}